Skip to content

Commit

Permalink
Fix practical issues with klamm sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@timwekkenbc
timwekkenbc committed Oct 17, 2024
1 parent c965676 commit 02f7e70
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 70 deletions.
60 changes: 24 additions & 36 deletions src/api/klamm/klamm.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ describe('KlammService', () => {
});

it('should initialize module correctly', async () => {
jest.spyOn(service as any, '_getUpdatedFilesFromGithub').mockResolvedValue(['file1.js']);
jest
.spyOn(service as any, '_getUpdatedFilesFromGithub')
.mockResolvedValue({ updatedFilesSinceLastDeploy: ['file1.js'], lastCommitAsyncTimestamp: 0 });
jest.spyOn(service as any, '_syncRules').mockResolvedValue(undefined);
jest.spyOn(service as any, '_updateLastSyncTimestamp').mockResolvedValue(undefined);

Expand Down Expand Up @@ -87,7 +89,10 @@ describe('KlammService', () => {
it('should get updated files from GitHub correctly', async () => {
const mockFiles = ['file1.js', 'file2.js'];
const mockCommits = [{ url: 'commit1' }, { url: 'commit2' }];
const mockCommitDetails = { files: [{ filename: 'rules/file1.js' }, { filename: 'rules/file2.js' }] };
const mockCommitDetails = {
files: [{ filename: 'rules/file1.js' }, { filename: 'rules/file2.js' }],
commit: { author: { date: 1234567790 } },
};
jest.spyOn(service as any, '_getLastSyncTimestamp').mockResolvedValue(1234567890);
jest
.spyOn(service.axiosGithubInstance, 'get')
Expand All @@ -101,7 +106,7 @@ describe('KlammService', () => {
expect(service.axiosGithubInstance.get).toHaveBeenCalledWith(
`${GITHUB_RULES_REPO}/commits?since=${new Date(1234567890).toISOString().split('.')[0]}Z&sha=${process.env.GITHUB_RULES_BRANCH}`,
);
expect(result).toEqual(mockFiles);
expect(result.updatedFilesSinceLastDeploy).toEqual(mockFiles);
});

it('should handle error in _getUpdatedFilesFromGithub', async () => {
Expand Down Expand Up @@ -143,12 +148,13 @@ describe('KlammService', () => {
const mockRule = { name: 'rule1', filepath: 'file1.js' } as RuleData;
const mockInputsOutputs = { inputs: [], resultOutputs: [] };
jest.spyOn(ruleMappingService, 'inputOutputSchemaFile').mockResolvedValue(mockInputsOutputs);
jest.spyOn(service as any, '_getFieldsFromIds').mockResolvedValue([]);
jest.spyOn(service as any, '_getAllKlammFields').mockResolvedValue([]);
jest.spyOn(service as any, '_getFieldsFromIds').mockReturnValue([]);

const result = await service['_getInputOutputFieldsData'](mockRule);

expect(ruleMappingService.inputOutputSchemaFile).toHaveBeenCalledWith(mockRule.filepath);
expect(service['_getFieldsFromIds']).toHaveBeenCalledWith([]);
expect(service['_getFieldsFromIds']).toHaveBeenCalledWith([], []);
expect(result).toEqual({ inputs: [], outputs: [] });
});

Expand All @@ -163,36 +169,16 @@ describe('KlammService', () => {

it('should get fields from IDs correctly', async () => {
const mockIds = [1, 2, 3];
jest.spyOn(service as any, '_fetchFieldById').mockResolvedValue({});

const result = await service['_getFieldsFromIds'](mockIds);

expect(service['_fetchFieldById']).toHaveBeenCalledTimes(mockIds.length);
expect(result).toEqual([{}, {}, {}]);
});
const mockFields = [
{ id: 1, name: 'field1', label: 'Field 1', description: 'Description 1' },
{ id: 2, name: 'field2', label: 'Field 2', description: 'Description 2' },
{ id: 3, name: 'field3', label: 'Field 3', description: 'Description 3' },
];
jest.spyOn(service as any, '_getAllKlammFields').mockResolvedValue(mockFields);

it('should handle error in _getFieldsFromIds', async () => {
const mockIds = [1, 2, 3];
jest.spyOn(service as any, '_fetchFieldById').mockRejectedValue(new Error('Error'));
const result = service['_getFieldsFromIds'](mockFields, mockIds);

await expect(service['_getFieldsFromIds'](mockIds)).rejects.toThrow('Error fetching fields by IDs: Error');
});

it('should fetch field by ID correctly', async () => {
const mockId = 1;
jest.spyOn(service.axiosKlammInstance, 'get').mockResolvedValue({ data: {} });

const result = await service['_fetchFieldById'](mockId);

expect(service.axiosKlammInstance.get).toHaveBeenCalledWith(`${process.env.KLAMM_API_URL}/api/brerules/${mockId}`);
expect(result).toEqual({});
});

it('should handle error in _fetchFieldById', async () => {
const mockId = 1;
jest.spyOn(service.axiosKlammInstance, 'get').mockRejectedValue(new Error('Error'));

await expect(service['_fetchFieldById'](mockId)).rejects.toThrow('Error fetching field with ID 1: Error');
expect(result).toEqual(mockFields);
});

it('should get child rules correctly', async () => {
Expand Down Expand Up @@ -299,19 +285,21 @@ describe('KlammService', () => {
it('should update last sync timestamp correctly', async () => {
jest.spyOn(klammSyncMetadata, 'findOneAndUpdate').mockResolvedValue(undefined);

await service['_updateLastSyncTimestamp']();
await service['_updateLastSyncTimestamp'](1234567890);

expect(klammSyncMetadata.findOneAndUpdate).toHaveBeenCalledWith(
{ key: 'singleton' },
{ lastSyncTimestamp: expect.any(Number) },
{ lastSyncTimestamp: 1234567890 },
{ upsert: true, new: true },
);
});

it('should handle error in _updateLastSyncTimestamp', async () => {
jest.spyOn(klammSyncMetadata, 'findOneAndUpdate').mockRejectedValue(new Error('Error'));

await expect(service['_updateLastSyncTimestamp']()).rejects.toThrow('Failed to update last sync timestamp');
await expect(service['_updateLastSyncTimestamp'](1234567890)).rejects.toThrow(
'Failed to update last sync timestamp',
);
});

it('should get last sync timestamp correctly', async () => {
Expand Down
70 changes: 39 additions & 31 deletions src/api/klamm/klamm.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,18 @@ export class KlammService {
async onModuleInit() {
try {
console.info('Syncing existing rules with Klamm...');
const updatedFilesSinceLastDeploy = await this._getUpdatedFilesFromGithub();
console.info('Files updated since last deploy:', updatedFilesSinceLastDeploy);
await this._syncRules(updatedFilesSinceLastDeploy);
console.info('Completed syncing existing rules with Klamm');
await this._updateLastSyncTimestamp();
const { updatedFilesSinceLastDeploy, lastCommitAsyncTimestamp } = await this._getUpdatedFilesFromGithub();
if (lastCommitAsyncTimestamp != undefined) {
console.info(
`Files updated since last deploy up to ${new Date(lastCommitAsyncTimestamp)}:`,
updatedFilesSinceLastDeploy,
);
await this._syncRules(updatedFilesSinceLastDeploy);
console.info('Completed syncing existing rules with Klamm');
await this._updateLastSyncTimestamp(lastCommitAsyncTimestamp);
} else {
console.info('Klamm file syncing up to date');
}
} catch (error) {
console.error('Unable to sync latest updates to Klamm:', error.message);
}
Expand Down Expand Up @@ -105,17 +112,22 @@ export class KlammService {
}
}

async _getUpdatedFilesFromGithub(): Promise<string[]> {
async _getUpdatedFilesFromGithub(): Promise<{
updatedFilesSinceLastDeploy: string[];
lastCommitAsyncTimestamp: number;
}> {
try {
// Get last updated time from from db
const timestamp = await this._getLastSyncTimestamp();
const date = new Date(timestamp);
const formattedDate = date.toISOString().split('.')[0] + 'Z'; // Format required for github api
console.log(`Getting files from Github from ${formattedDate} onwards...`);
// Fetch commits since the specified timestamp
const commitsResponse = await this.axiosGithubInstance.get(
`${GITHUB_RULES_REPO}/commits?since=${formattedDate}&sha=${process.env.GITHUB_RULES_BRANCH}`,
);
const commits = commitsResponse.data;
const commits = commitsResponse.data.reverse();
let lastCommitAsyncTimestamp;
// Fetch details for each commit to get the list of changed files
const updatedFiles = new Set<string>();
for (const commit of commits) {
Expand All @@ -128,8 +140,9 @@ export class KlammService {
}
}
}
lastCommitAsyncTimestamp = new Date(commitDetails.commit.author.date).getTime() + 1000;
}
return Array.from(updatedFiles);
return { updatedFilesSinceLastDeploy: Array.from(updatedFiles), lastCommitAsyncTimestamp };
} catch (error) {
console.error('Error fetching updated files from GitHub:', error);
throw new Error('Error fetching updated files from GitHub');
Expand Down Expand Up @@ -163,37 +176,33 @@ export class KlammService {
const { inputs, resultOutputs } = await this.ruleMappingService.inputOutputSchemaFile(rule.filepath);
const inputIds = inputs.map(({ id }) => Number(id));
const outputIds = resultOutputs.map(({ id }) => Number(id));
const inputResults = await this._getFieldsFromIds(inputIds);
const outputResults = await this._getFieldsFromIds(outputIds);
const klammFields: KlammField[] = await this._getAllKlammFields();
const inputResults = this._getFieldsFromIds(klammFields, inputIds);
const outputResults = this._getFieldsFromIds(klammFields, outputIds);
return { inputs: inputResults, outputs: outputResults };
} catch (error) {
console.error(`Error getting input/output fields for rule ${rule.name}`, error.message);
throw new Error(`Error getting input/output fields for rule ${rule.name}`);
}
}

async _getFieldsFromIds(ids: number[]): Promise<any[]> {
try {
const promises = ids.map((id) => this._fetchFieldById(id));
return await Promise.all(promises);
} catch (error) {
console.error(`Error fetching fields by IDs`, error.message);
throw new Error(`Error fetching fields by IDs: ${error.message}`);
}
_getFieldsFromIds(klammFields: KlammField[], ids: number[]): KlammField[] {
const fieldObjects: KlammField[] = [];
klammFields.forEach((fieldObject) => {
if (ids.includes(fieldObject.id)) {
fieldObjects.push(fieldObject);
}
});
return fieldObjects;
}

private async _fetchFieldById(id: number): Promise<any> {
async _getAllKlammFields(): Promise<KlammField[]> {
try {
const response = await this.axiosKlammInstance.get(`${process.env.KLAMM_API_URL}/api/brerules/${id}`);
return response.data;
const response = await this.axiosKlammInstance.get(`${process.env.KLAMM_API_URL}/api/brerules`);
return response.data.data;
} catch (error) {
if (error.response && error.response.status === 404) {
console.warn(`Field with ID ${id} not found`);
return null;
} else {
console.error(`Error fetching field with ID ${id}`, error.message);
throw new Error(`Error fetching field with ID ${id}: ${error.message}`);
}
console.error('Error fetching fields from Klamm', error.message);
throw new Error(`Error fetching fields from Klamm: ${error.message}`);
}
}

Expand Down Expand Up @@ -253,12 +262,11 @@ export class KlammService {
}
}

async _updateLastSyncTimestamp(): Promise<void> {
async _updateLastSyncTimestamp(lastSyncTimestamp: number): Promise<void> {
try {
const timestamp = Date.now();
await this.klammSyncMetadata.findOneAndUpdate(
{ key: 'singleton' },
{ lastSyncTimestamp: timestamp },
{ lastSyncTimestamp },
{ upsert: true, new: true },
);
} catch (error) {
Expand Down
3 changes: 0 additions & 3 deletions src/api/ruleData/ruleData.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,6 @@ export class RuleDataService {
async getRuleDataByFilepath(filepath: string): Promise<RuleData> {
try {
const ruleData = await this.ruleDataModel.findOne({ filepath }).exec();
if (!ruleData) {
throw new Error('Rule data not found');
}
return ruleData;
} catch (error) {
throw new Error(`Error getting all rule data for ${filepath}: ${error.message}`);
Expand Down

0 comments on commit 02f7e70

Please sign in to comment.