diff --git a/api-gateway/src/api/service/schema.ts b/api-gateway/src/api/service/schema.ts index e2f2aa7d23..76218c6955 100644 --- a/api-gateway/src/api/service/schema.ts +++ b/api-gateway/src/api/service/schema.ts @@ -1925,13 +1925,21 @@ export class SchemaApi { const ids = schemas.map(s => s.id); const tags = await guardians.exportTags(owner, 'Schema', ids); const name = `${Date.now()}`; - const zip = await SchemaImportExport.generateZipFile({ schemas, tags }); + + const zip = await SchemaImportExport.generateZipFile({ + schemas, + tags, + helpers: guardians, + user + }); + const arcStream = zip.generateNodeStream({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 3 - } + }, + platform: 'UNIX', }); res.header('Content-disposition', `attachment; filename=${name}`); res.header('Content-type', 'application/zip'); diff --git a/common/src/entity/formula.ts b/common/src/entity/formula.ts index 99461dd883..ae0c88c393 100644 --- a/common/src/entity/formula.ts +++ b/common/src/entity/formula.ts @@ -1,6 +1,8 @@ -import { BeforeCreate, Entity, Property } from '@mikro-orm/core'; +import {AfterDelete, BeforeCreate, Entity, Property} from '@mikro-orm/core'; import { BaseEntity } from '../models/index.js'; import { EntityStatus, GenerateUUIDv4, IFormula, IFormulaConfig } from '@guardian/interfaces'; +import {ObjectId} from "@mikro-orm/mongodb"; +import {DataBaseHelper} from "../helpers/index.js"; /** * Formula collection @@ -85,6 +87,12 @@ export class Formula extends BaseEntity implements IFormula { @Property({ nullable: true, type: 'unknown' }) config?: IFormulaConfig; + /** + * File id of the original formula zip (publish flow). + */ + @Property({ nullable: true }) + contentFileId?: ObjectId; + /** * Set defaults */ @@ -93,4 +101,16 @@ export class Formula extends BaseEntity implements IFormula { this.uuid = this.uuid || GenerateUUIDv4(); this.status = this.status || EntityStatus.DRAFT; } -} \ No newline at end of file + + @AfterDelete() + deleteContentFile() { + if (this.contentFileId) { + DataBaseHelper.gridFS + .delete(this.contentFileId) + .catch((reason) => { + console.error('AfterDelete: Formula, contentFileId'); + console.error(reason); + }); + } + } +} diff --git a/common/src/entity/module.ts b/common/src/entity/module.ts index 1143f6be10..5d57b4c701 100644 --- a/common/src/entity/module.ts +++ b/common/src/entity/module.ts @@ -94,6 +94,12 @@ export class PolicyModule extends BaseEntity { @Property({ persist: false, nullable: true }) _configFileId?: ObjectId; + /** + * File id of the original module zip (publish flow). + */ + @Property({ nullable: true }) + contentFileId?: ObjectId; + /** * Set defaults */ @@ -169,4 +175,19 @@ export class PolicyModule extends BaseEntity { }); } } + + /** + * Delete original module zip (publish flow) + */ + @AfterDelete() + deleteContentFile() { + if (this.contentFileId) { + DataBaseHelper.gridFS + .delete(this.contentFileId) + .catch((reason) => { + console.error('AfterDelete: PolicyModule, contentFileId'); + console.error(reason); + }); + } + } } diff --git a/common/src/entity/policy-label.ts b/common/src/entity/policy-label.ts index f42b83dde1..5c57b85d6d 100644 --- a/common/src/entity/policy-label.ts +++ b/common/src/entity/policy-label.ts @@ -1,6 +1,8 @@ -import { BeforeCreate, Entity, Property } from '@mikro-orm/core'; +import {AfterDelete, BeforeCreate, Entity, Property} from '@mikro-orm/core'; import { BaseEntity } from '../models/index.js'; import { EntityStatus, GenerateUUIDv4, IPolicyLabel, IPolicyLabelConfig } from '@guardian/interfaces'; +import { DataBaseHelper } from '../helpers/db-helper.js'; +import { ObjectId } from "@mikro-orm/mongodb"; /** * PolicyLabel collection @@ -100,6 +102,12 @@ export class PolicyLabel extends BaseEntity implements IPolicyLabel { @Property({ nullable: true }) method?: string; + /** + * File id of the original Policy Label zip (publish flow). + */ + @Property({ nullable: true }) + contentFileId?: ObjectId; + /** * Set defaults */ @@ -108,4 +116,19 @@ export class PolicyLabel extends BaseEntity implements IPolicyLabel { this.uuid = this.uuid || GenerateUUIDv4(); this.status = this.status || EntityStatus.DRAFT; } + + /** + * Delete original Policy Label zip (publish flow) + */ + @AfterDelete() + deleteContentFile() { + if (this.contentFileId) { + DataBaseHelper.gridFS + .delete(this.contentFileId) + .catch((reason) => { + console.error('AfterDelete: PolicyLabel, contentFileId'); + console.error(reason); + }); + } + } } diff --git a/common/src/entity/policy-statistic.ts b/common/src/entity/policy-statistic.ts index da91fbee07..6f0dfca6c2 100644 --- a/common/src/entity/policy-statistic.ts +++ b/common/src/entity/policy-statistic.ts @@ -1,6 +1,8 @@ -import { BeforeCreate, Entity, Property } from '@mikro-orm/core'; +import {AfterDelete, BeforeCreate, Entity, Property} from '@mikro-orm/core'; import { BaseEntity } from '../models/index.js'; import { EntityStatus, GenerateUUIDv4, IStatistic, IStatisticConfig } from '@guardian/interfaces'; +import {ObjectId} from "@mikro-orm/mongodb"; +import { DataBaseHelper } from '../helpers/index.js'; /** * PolicyStatistic collection @@ -100,6 +102,12 @@ export class PolicyStatistic extends BaseEntity implements IStatistic { @Property({ nullable: true }) method?: string; + /** + * File id of the original Policy Statistic file (publish flow). + */ + @Property({ nullable: true }) + contentFileId?: ObjectId; + /** * Set defaults */ @@ -108,4 +116,19 @@ export class PolicyStatistic extends BaseEntity implements IStatistic { this.uuid = this.uuid || GenerateUUIDv4(); this.status = this.status || EntityStatus.DRAFT; } + + /** + * Delete original Policy Statistic file (publish flow) + */ + @AfterDelete() + deleteContentFile() { + if (this.contentFileId) { + DataBaseHelper.gridFS + .delete(this.contentFileId) + .catch((reason) => { + console.error('AfterDelete: PolicyStatistic, contentFileId'); + console.error(reason); + }); + } + } } diff --git a/common/src/entity/policy.ts b/common/src/entity/policy.ts index cc3af0145b..c4e20a4716 100644 --- a/common/src/entity/policy.ts +++ b/common/src/entity/policy.ts @@ -296,6 +296,13 @@ export class Policy extends BaseEntity { @Property({ nullable: true }) originalMessageId?: string; + + /** + * File id of the original policy zip (publish flow). + */ + @Property({ nullable: true }) + contentFileId?: ObjectId; + /** * Set policy defaults */ @@ -407,4 +414,19 @@ export class Policy extends BaseEntity { }); } } + + /** + * Delete original policy zip (publish flow) + */ + @AfterDelete() + deleteContentFile() { + if (this.contentFileId) { + DataBaseHelper.gridFS + .delete(this.contentFileId) + .catch((reason) => { + console.error('AfterDelete: Policy, contentFileId'); + console.error(reason); + }); + } + } } diff --git a/common/src/entity/schema.ts b/common/src/entity/schema.ts index a427eb47c4..f81d22d3ee 100644 --- a/common/src/entity/schema.ts +++ b/common/src/entity/schema.ts @@ -193,6 +193,18 @@ export class Schema extends BaseEntity implements ISchema { @Property({ persist: false, nullable: true }) _contextFileId?: ObjectId; + /** + * Document file id of the original schema(publish flow). + */ + @Property({ nullable: true }) + contentDocumentFileId?: string; + + /** + * Context file id of the original schema(publish flow). + */ + @Property({ nullable: true }) + contentContextFileId?: string; + /** * Schema defaults */ @@ -323,4 +335,27 @@ export class Schema extends BaseEntity implements ISchema { }); } } + + /** + * Delete original schema document and context(publish flow) + */ + @AfterDelete() + deleteContentFiles() { + if (this.contentDocumentFileId) { + DataBaseHelper.gridFS + .delete(new ObjectId(this.contentDocumentFileId)) + .catch((reason) => { + console.error('AfterDelete: Schema, contentDocumentFileId'); + console.error(reason); + }); + } + if (this.contentContextFileId) { + DataBaseHelper.gridFS + .delete(new ObjectId(this.contentContextFileId)) + .catch((reason) => { + console.error('AfterDelete: Schema, contentContextFileId'); + console.error(reason); + }); + } + } } diff --git a/common/src/entity/tag.ts b/common/src/entity/tag.ts index 3f31f9d91a..5254ec92e7 100644 --- a/common/src/entity/tag.ts +++ b/common/src/entity/tag.ts @@ -3,6 +3,7 @@ import { RestoreEntity } from '../models/index.js'; import { GenerateUUIDv4 } from '@guardian/interfaces'; import { DataBaseHelper } from '../helpers/db-helper.js'; import { DeleteCache } from './delete-cache.js'; +import { ObjectId } from "@mikro-orm/mongodb"; /** * Tags collection @@ -102,6 +103,12 @@ export class Tag extends RestoreEntity { @Property({ nullable: false }) date: string; + /** + * File id of the original tag (publish flow). + */ + @Property({ nullable: true }) + contentFileId?: string; + /** * Create document */ @@ -151,4 +158,19 @@ export class Tag extends RestoreEntity { console.error(error); } } + + /** + * Delete original tag file (publish flow) + */ + @AfterDelete() + deleteContentFile() { + if (this.contentFileId) { + DataBaseHelper.gridFS + .delete(new ObjectId(this.contentFileId)) + .catch((reason) => { + console.error('AfterDelete: Tag, contentFileId'); + console.error(reason); + }); + } + } } diff --git a/common/src/entity/tool.ts b/common/src/entity/tool.ts index 13725d4566..b106aaf786 100644 --- a/common/src/entity/tool.ts +++ b/common/src/entity/tool.ts @@ -111,6 +111,12 @@ export class PolicyTool extends BaseEntity { @Property({ persist: false, nullable: true }) _configFileId?: ObjectId; + /** + * File id of the original tool zip (imported from IPFS or publish flow). + */ + @Property({ nullable: true }) + contentFileId?: ObjectId; + /** * Set defaults */ @@ -186,4 +192,16 @@ export class PolicyTool extends BaseEntity { }); } } + + @AfterDelete() + deleteContentFile() { + if (this.contentFileId) { + DataBaseHelper.gridFS + .delete(this.contentFileId) + .catch((reason) => { + console.error('AfterDelete: PolicyTool, contentFileId'); + console.error(reason); + }); + } + } } diff --git a/common/src/import-export/formula.ts b/common/src/import-export/formula.ts index 56e76c783d..e84a7735d1 100644 --- a/common/src/import-export/formula.ts +++ b/common/src/import-export/formula.ts @@ -2,6 +2,7 @@ import JSZip from 'jszip'; import { Formula, Policy } from '../entity/index.js'; import { IFormulaConfig } from '@guardian/interfaces'; import { DatabaseServer } from '../database-modules/index.js'; +import { ImportExportUtils } from './utils.js'; interface ISchemaComponents { iri: string; @@ -90,8 +91,11 @@ export class FormulaImportExport { delete formulas.updateDate; const schemas = components.schemas; const zip = new JSZip(); - zip.file(FormulaImportExport.schemasFileName, JSON.stringify(schemas)); - zip.file(FormulaImportExport.formulaFileName, JSON.stringify(formulas)); + + const ZIP_FILE_OPTIONS = ImportExportUtils.getDeterministicZipFileOptions(); + + zip.file(FormulaImportExport.schemasFileName, JSON.stringify(schemas), ZIP_FILE_OPTIONS); + zip.file(FormulaImportExport.formulaFileName, JSON.stringify(formulas), ZIP_FILE_OPTIONS); return zip; } diff --git a/common/src/import-export/module.ts b/common/src/import-export/module.ts index 9e53915b34..a3ce0b9f5c 100644 --- a/common/src/import-export/module.ts +++ b/common/src/import-export/module.ts @@ -1,6 +1,7 @@ import JSZip from 'jszip'; import { PolicyModule, Schema, Tag } from '../entity/index.js'; import { DatabaseServer } from '../database-modules/index.js'; +import { ImportExportUtils } from './utils.js'; /** * Module components @@ -73,18 +74,20 @@ export class ModuleImportExport { const zip = new JSZip(); - zip.file(ModuleImportExport.moduleFileName, JSON.stringify(moduleObject)); + const ZIP_FILE_OPTIONS = ImportExportUtils.getDeterministicZipFileOptions(); - zip.folder('tags'); + zip.file(ModuleImportExport.moduleFileName, JSON.stringify(moduleObject), ZIP_FILE_OPTIONS); + + ImportExportUtils.addDeterministicZipDir(zip, 'tags'); for (let index = 0; index < components.tags.length; index++) { const tag = { ...components.tags[index] }; delete tag.id; delete tag._id; tag.status = 'History'; - zip.file(`tags/${index}.json`, JSON.stringify(tag)); + zip.file(`tags/${index}.json`, JSON.stringify(tag), ZIP_FILE_OPTIONS); } - zip.folder('schemas'); + ImportExportUtils.addDeterministicZipDir(zip, 'schemas'); for (const schema of components.schemas) { const item = { ...schema }; delete item._id; @@ -92,7 +95,7 @@ export class ModuleImportExport { delete item.status; delete item.readonly; item.id = schema.id.toString(); - zip.file(`schemas/${item.iri}.json`, JSON.stringify(item)); + zip.file(`schemas/${item.iri}.json`, JSON.stringify(item), ZIP_FILE_OPTIONS); } return zip; diff --git a/common/src/import-export/policy-label.ts b/common/src/import-export/policy-label.ts index 0e0cb2b29f..615c03c2a5 100644 --- a/common/src/import-export/policy-label.ts +++ b/common/src/import-export/policy-label.ts @@ -19,6 +19,7 @@ import { import { PolicyStatisticImportExport } from './policy-statistic.js'; import { PolicyImportExport } from './policy.js'; import { DatabaseServer } from '../database-modules/index.js'; +import {ImportExportUtils} from './utils.js'; /** * PolicyLabel components @@ -72,7 +73,10 @@ export class PolicyLabelImportExport { delete object.createDate; delete object.updateDate; const zip = new JSZip(); - zip.file(PolicyLabelImportExport.fileName, JSON.stringify(object)); + + const ZIP_FILE_OPTIONS = ImportExportUtils.getDeterministicZipFileOptions(); + zip.file(PolicyLabelImportExport.fileName, JSON.stringify(object), ZIP_FILE_OPTIONS); + return zip; } diff --git a/common/src/import-export/policy-statistic.ts b/common/src/import-export/policy-statistic.ts index 2e64bcb1f9..3948f92114 100644 --- a/common/src/import-export/policy-statistic.ts +++ b/common/src/import-export/policy-statistic.ts @@ -1,9 +1,21 @@ import JSZip from 'jszip'; import { Policy, PolicyStatistic, Schema as SchemaCollection } from '../entity/index.js'; -import { IFormulaData, IRuleData, IScoreData, IScoreOption, IStatisticConfig, IVariableData, Schema, SchemaEntity, SchemaStatus } from '@guardian/interfaces'; +import { + EntityStatus, + IFormulaData, + IRuleData, + IScoreData, + IScoreOption, + IStatisticConfig, + IVariableData, + Schema, + SchemaEntity, + SchemaStatus +} from '@guardian/interfaces'; import { SchemaRuleImportExport } from './schema-rule.js'; import { PolicyImportExport } from './policy.js'; import { DatabaseServer } from '../database-modules/index.js'; +import { ImportExportUtils } from './utils.js'; /** * PolicyStatistic components @@ -57,7 +69,16 @@ export class PolicyStatisticImportExport { delete object.createDate; delete object.updateDate; const zip = new JSZip(); - zip.file(PolicyStatisticImportExport.policyStatisticFileName, JSON.stringify(object)); + const ZIP_FILE_OPTIONS = ImportExportUtils.getDeterministicZipFileOptions(); + zip.file(PolicyStatisticImportExport.policyStatisticFileName, JSON.stringify(object), ZIP_FILE_OPTIONS); + + if (object.status === EntityStatus.PUBLISHED && object.contentFileId) { + ImportExportUtils.addDeterministicZipDir(zip, 'ipfs'); + + const buffer = await DatabaseServer.loadFile(object.contentFileId); + zip.file(`ipfs/${object.uuid}.json`, Buffer.from(buffer), ZIP_FILE_OPTIONS); + } + return zip; } diff --git a/common/src/import-export/schema.ts b/common/src/import-export/schema.ts index 20e72d121f..962d2654d3 100644 --- a/common/src/import-export/schema.ts +++ b/common/src/import-export/schema.ts @@ -1,6 +1,8 @@ import JSZip from 'jszip'; import { Tag } from '../entity/index.js'; import { ISchema } from '@guardian/interfaces'; +import { ImportExportUtils } from './utils.js'; +import { IAuthUser } from "../interfaces"; /** * Schema components @@ -8,6 +10,8 @@ import { ISchema } from '@guardian/interfaces'; export interface ISchemaComponents { schemas: ISchema[]; tags: Tag[]; + helpers?: Record; + user?: IAuthUser; } /** @@ -22,16 +26,46 @@ export class SchemaImportExport { */ public static async generateZipFile(components: ISchemaComponents): Promise { const zip = new JSZip(); + + const ZIP_FILE_OPTIONS = ImportExportUtils.getDeterministicZipFileOptions(); + for (const schema of components.schemas) { - zip.file(`${schema.iri}.json`, JSON.stringify(schema)); + zip.file(`${schema.iri}.json`, JSON.stringify(schema), ZIP_FILE_OPTIONS); } if (Array.isArray(components.tags)) { - zip.folder('tags') + ImportExportUtils.addDeterministicZipDir(zip, 'tags'); for (let index = 0; index < components.tags.length; index++) { const tag = components.tags[index]; - zip.file(`tags/${index}.json`, JSON.stringify(tag)); + zip.file(`tags/${index}.json`, JSON.stringify(tag), ZIP_FILE_OPTIONS); + } + } + + if (components.helpers && components.user) { + ImportExportUtils.addDeterministicZipDir(zip, 'ipfs'); + for (const schema of components.schemas) { + if (schema.status === 'PUBLISHED' && schema.contentDocumentFileId) { + const doc = await components.helpers.csvGetFile(schema.contentDocumentFileId.toString(), components.user); + + zip.file(`ipfs/${schema.iri}.document.json`, Buffer.from(doc.buffer.data), ZIP_FILE_OPTIONS); + } + if (schema.status === 'PUBLISHED' && schema.contentContextFileId) { + const ctx = await components.helpers.csvGetFile(schema.contentContextFileId.toString(), components.user); + + zip.file(`ipfs/${schema.iri}.context.json`, Buffer.from(ctx.buffer.data), ZIP_FILE_OPTIONS); + } + } + + if (Array.isArray(components.tags)) { + ImportExportUtils.addDeterministicZipDir(zip, 'ipfs/tags'); + for (const tag of components.tags) { + if (tag.contentFileId) { + const doc = await components.helpers.csvGetFile(tag.contentFileId.toString(), components.user); + zip.file(`ipfs/tags/${tag.uuid}.json`, Buffer.from(doc.buffer.data), ZIP_FILE_OPTIONS); + } + } } } + return zip; } @@ -46,6 +80,7 @@ export class SchemaImportExport { const schemaStringArray = await Promise.all(Object.entries(content.files) .filter(file => !file[1].dir) .filter(file => !/^tags\/.+/.test(file[0])) + .filter(file => !/^ipfs\/.+/.test(file[0])) .map(file => file[1].async('string'))); const schemas = schemaStringArray.map(item => JSON.parse(item)); diff --git a/common/src/import-export/tool.ts b/common/src/import-export/tool.ts index 6e976808f1..fd09a9d664 100644 --- a/common/src/import-export/tool.ts +++ b/common/src/import-export/tool.ts @@ -78,18 +78,20 @@ export class ToolImportExport { const zip = new JSZip(); - zip.file(ToolImportExport.toolFileName, JSON.stringify(toolObject)); + const ZIP_FILE_OPTIONS = ImportExportUtils.getDeterministicZipFileOptions(); - zip.folder('tags'); + zip.file(ToolImportExport.toolFileName, JSON.stringify(toolObject), ZIP_FILE_OPTIONS); + + ImportExportUtils.addDeterministicZipDir(zip, 'tags'); for (let index = 0; index < components.tags.length; index++) { const tag = { ...components.tags[index] }; delete tag.id; delete tag._id; tag.status = 'History'; - zip.file(`tags/${index}.json`, JSON.stringify(tag)); + zip.file(`tags/${index}.json`, JSON.stringify(tag), ZIP_FILE_OPTIONS); } - zip.folder('schemas'); + ImportExportUtils.addDeterministicZipDir(zip, 'schemas'); for (const schema of components.schemas) { const item = { ...schema }; delete item._id; @@ -97,10 +99,10 @@ export class ToolImportExport { delete item.status; delete item.readonly; item.id = schema.id.toString(); - zip.file(`schemas/${item.iri}.json`, JSON.stringify(item)); + zip.file(`schemas/${item.iri}.json`, JSON.stringify(item), ZIP_FILE_OPTIONS); } - zip.folder('tools'); + ImportExportUtils.addDeterministicZipDir(zip, 'tools'); for (const tool of components.tools) { const item = { name: tool.name, @@ -109,7 +111,7 @@ export class ToolImportExport { owner: tool.creator, hash: tool.hash }; - zip.file(`tools/${tool.hash}.json`, JSON.stringify(item)); + zip.file(`tools/${tool.hash}.json`, JSON.stringify(item), ZIP_FILE_OPTIONS); } return zip; diff --git a/common/src/import-export/utils.ts b/common/src/import-export/utils.ts index 86252e681c..9b761abfe8 100644 --- a/common/src/import-export/utils.ts +++ b/common/src/import-export/utils.ts @@ -1,5 +1,6 @@ import { BlockType } from '@guardian/interfaces'; import { SchemaFields, TokenFields } from '../helpers/index.js'; +import JSZip from 'jszip'; interface IBlockConfig { blockType: string; @@ -12,6 +13,8 @@ interface IBlockConfig { * Import\Export utils */ export class ImportExportUtils { + public static readonly DETERMINISTIC_ZIP_DATE = new Date(Date.UTC(1980, 0, 1, 0, 0, 0)); + /** * Find all tools * @param config @@ -132,4 +135,32 @@ export class ImportExportUtils { const map = finder(obj, new Set()); return Array.from(map); } + + /** + * Get Deterministic Zip File Options + */ + public static getDeterministicZipFileOptions() { + return { + createFolders: false, + date: ImportExportUtils.DETERMINISTIC_ZIP_DATE, + unixPermissions: 0o100644, + dosPermissions: 0x20, + }; + } + + /** + * Add Deterministic Zip Dir + * @param zip + * @param name + */ + public static addDeterministicZipDir(zip: JSZip, name: string): void { + const dirName = `${name}/`; + + zip.file(dirName, '', { + dir: true, + date: ImportExportUtils.DETERMINISTIC_ZIP_DATE, + unixPermissions: 0o040755, + dosPermissions: 0x10, + }); + } } diff --git a/guardian-service/src/api/formulas.service.ts b/guardian-service/src/api/formulas.service.ts index 22a839506a..2e5d846b10 100644 --- a/guardian-service/src/api/formulas.service.ts +++ b/guardian-service/src/api/formulas.service.ts @@ -234,6 +234,13 @@ export async function formulasAPI(logger: PinoLogger): Promise { return new MessageError('Item does not exist.'); } + if (item.status === EntityStatus.PUBLISHED && item.contentFileId) { + const buffer = await DatabaseServer.loadFile(item.contentFileId); + const arrayBuffer = Uint8Array.from(buffer).buffer; + + return new BinaryMessageResponse(arrayBuffer); + } + const zip = await FormulaImportExport.generate(item); const file = await zip.generateAsync({ type: 'arraybuffer', @@ -241,6 +248,7 @@ export async function formulasAPI(logger: PinoLogger): Promise { compressionOptions: { level: 3, }, + platform: 'UNIX', }); return new BinaryMessageResponse(file); diff --git a/guardian-service/src/api/helpers/formulas-helpers.ts b/guardian-service/src/api/helpers/formulas-helpers.ts index abc3da54c2..8059605275 100644 --- a/guardian-service/src/api/helpers/formulas-helpers.ts +++ b/guardian-service/src/api/helpers/formulas-helpers.ts @@ -1,5 +1,5 @@ import { DatabaseServer, Formula, FormulaImportExport, FormulaMessage, INotificationStep, MessageAction, MessageServer, TopicConfig, VcDocument, VpDocument } from '@guardian/common'; -import { EntityStatus, IOwner, IRootConfig } from '@guardian/interfaces'; +import {EntityStatus, GenerateUUIDv4, IOwner, IRootConfig} from '@guardian/interfaces'; type IDocument = VcDocument | VpDocument; @@ -100,11 +100,13 @@ export async function publishFormula( // <-- Steps const STEP_RESOLVE_TOPIC = 'Resolve topic'; + const STEP_SAVE_FILE_IN_DB = 'Save file in database'; const STEP_PUBLISH_FORMULA = 'Publish formula'; // Steps --> - notifier.addStep(STEP_RESOLVE_TOPIC, 30); - notifier.addStep(STEP_PUBLISH_FORMULA, 70); + notifier.addStep(STEP_RESOLVE_TOPIC, 25); + notifier.addStep(STEP_SAVE_FILE_IN_DB, 10); + notifier.addStep(STEP_PUBLISH_FORMULA, 65); notifier.start(); notifier.startStep(STEP_RESOLVE_TOPIC); @@ -116,16 +118,22 @@ export async function publishFormula( }).setTopicObject(topic); notifier.completeStep(STEP_RESOLVE_TOPIC); - notifier.startStep(STEP_PUBLISH_FORMULA); const zip = await FormulaImportExport.generate(item); const buffer = await zip.generateAsync({ type: 'arraybuffer', compression: 'DEFLATE', compressionOptions: { level: 3 - } + }, + platform: 'UNIX', }); + notifier.startStep(STEP_SAVE_FILE_IN_DB); + item.contentFileId = await DatabaseServer.saveFile(GenerateUUIDv4(), Buffer.from(buffer)); + notifier.completeStep(STEP_SAVE_FILE_IN_DB); + + notifier.startStep(STEP_PUBLISH_FORMULA); + const publishMessage = new FormulaMessage(MessageAction.PublishFormula); publishMessage.setDocument(item, buffer); const statMessageResult = await messageServer diff --git a/guardian-service/src/api/module.service.ts b/guardian-service/src/api/module.service.ts index 4fcf798f28..eaa4a3ea3b 100644 --- a/guardian-service/src/api/module.service.ts +++ b/guardian-service/src/api/module.service.ts @@ -169,6 +169,7 @@ export async function publishModule( const STEP_RESOLVE_TOPIC = 'Find topic'; const STEP_CREATE_TOPIC = 'Create module topic'; const STEP_GENERATE_FILE = 'Generate file'; + const STEP_SAVE_FILE_IN_DB = 'Save file in database'; const STEP_PUBLISH_MODULE = 'Publish module'; const STEP_LINK_TOPIC = 'Link topic and module'; const STEP_SAVE = 'Save'; @@ -178,6 +179,7 @@ export async function publishModule( notifier.addStep(STEP_RESOLVE_TOPIC); notifier.addStep(STEP_CREATE_TOPIC); notifier.addStep(STEP_GENERATE_FILE); + notifier.addStep(STEP_SAVE_FILE_IN_DB); notifier.addStep(STEP_PUBLISH_MODULE); notifier.addStep(STEP_LINK_TOPIC); notifier.addStep(STEP_SAVE); @@ -226,10 +228,15 @@ export async function publishModule( compression: 'DEFLATE', compressionOptions: { level: 3 - } + }, + platform: 'UNIX', }); notifier.completeStep(STEP_GENERATE_FILE); + notifier.startStep(STEP_SAVE_FILE_IN_DB); + model.contentFileId = await DatabaseServer.saveFile(GenerateUUIDv4(), Buffer.from(buffer)); + notifier.completeStep(STEP_SAVE_FILE_IN_DB); + notifier.startStep(STEP_PUBLISH_MODULE); const message = new ModuleMessage(MessageType.Module, MessageAction.PublishModule); message.setDocument(model, buffer); @@ -475,6 +482,13 @@ export async function modulesAPI(logger: PinoLogger): Promise { throw new Error('Invalid module'); } + if (item.status === ModuleStatus.PUBLISHED && item.contentFileId) { + const buffer = await DatabaseServer.loadFile(item.contentFileId); + const arrayBuffer = Uint8Array.from(buffer).buffer; + + return new BinaryMessageResponse(arrayBuffer); + } + updateModuleConfig(item); const zip = await ModuleImportExport.generate(item); const file = await zip.generateAsync({ @@ -483,6 +497,7 @@ export async function modulesAPI(logger: PinoLogger): Promise { compressionOptions: { level: 3, }, + platform: 'UNIX', }); return new BinaryMessageResponse(file); } catch (error) { diff --git a/guardian-service/src/api/policy-labels.service.ts b/guardian-service/src/api/policy-labels.service.ts index e9e7086b0f..ccab6cf043 100644 --- a/guardian-service/src/api/policy-labels.service.ts +++ b/guardian-service/src/api/policy-labels.service.ts @@ -18,7 +18,16 @@ import { INotificationStep, NewNotifier, } from '@guardian/common'; -import { EntityStatus, IOwner, LabelValidators, MessageAPI, PolicyStatus, Schema, SchemaStatus } from '@guardian/interfaces'; +import { + EntityStatus, + GenerateUUIDv4, + IOwner, + LabelValidators, + MessageAPI, + PolicyStatus, + Schema, + SchemaStatus +} from '@guardian/interfaces'; import { findRelationships, generateSchema, generateVpDocument, getOrCreateTopic, publishLabelConfig } from './helpers/policy-labels-helpers.js'; import { publishSchemas } from '../helpers/import-helpers/index.js'; @@ -86,9 +95,12 @@ async function publishPolicyLabel( compression: 'DEFLATE', compressionOptions: { level: 3 - } + }, + platform: 'UNIX', }); + item.contentFileId = await DatabaseServer.saveFile(GenerateUUIDv4(), Buffer.from(buffer)); + const statMessage = new LabelMessage(MessageAction.PublishPolicyLabel); statMessage.setDocument(item, buffer); const statMessageResult = await messageServer @@ -444,6 +456,12 @@ export async function policyLabelsAPI(logger: PinoLogger): Promise { return new MessageError('Item does not exist.'); } + if (item.status === EntityStatus.PUBLISHED && item.contentFileId) { + const buffer = await DatabaseServer.loadFile(item.contentFileId); + const arrayBuffer = Uint8Array.from(buffer).buffer; + return new BinaryMessageResponse(arrayBuffer); + } + const zip = await PolicyLabelImportExport.generate(item); const file = await zip.generateAsync({ type: 'arraybuffer', @@ -451,6 +469,7 @@ export async function policyLabelsAPI(logger: PinoLogger): Promise { compressionOptions: { level: 3, }, + platform: 'UNIX', }); return new BinaryMessageResponse(file); diff --git a/guardian-service/src/api/policy-statistics.service.ts b/guardian-service/src/api/policy-statistics.service.ts index f3c5238910..ad939ca99f 100644 --- a/guardian-service/src/api/policy-statistics.service.ts +++ b/guardian-service/src/api/policy-statistics.service.ts @@ -1,6 +1,14 @@ import { ApiResponse } from './helpers/api-response.js'; import { BinaryMessageResponse, DatabaseServer, MessageAction, MessageError, MessageResponse, MessageServer, NewNotifier, PinoLogger, PolicyStatistic, PolicyStatisticImportExport, StatisticAssessmentMessage, StatisticMessage, Users } from '@guardian/common'; -import { EntityStatus, IOwner, MessageAPI, PolicyStatus, Schema, SchemaEntity } from '@guardian/interfaces'; +import { + EntityStatus, + GenerateUUIDv4, + IOwner, + MessageAPI, + PolicyStatus, + Schema, + SchemaEntity +} from '@guardian/interfaces'; import { findRelationships, generateSchema, generateVcDocument, getOrCreateTopic, publishConfig, uniqueDocuments } from './helpers/policy-statistics-helpers.js'; import { publishSchema } from '../helpers/import-helpers/index.js'; @@ -290,6 +298,10 @@ export async function statisticsAPI(logger: PinoLogger): Promise { operatorKey: user.hederaAccountKey, signOptions: user.signOptions }); + + const buffer = Buffer.from(JSON.stringify(item.config)); + item.contentFileId = await DatabaseServer.saveFile(GenerateUUIDv4(), buffer); + const statMessageResult = await messageServer .setTopicObject(topic) .sendMessage(statMessage, { @@ -676,6 +688,7 @@ export async function statisticsAPI(logger: PinoLogger): Promise { compressionOptions: { level: 3, }, + platform: 'UNIX', }); return new BinaryMessageResponse(file); diff --git a/guardian-service/src/api/tool.service.ts b/guardian-service/src/api/tool.service.ts index d842746d57..d041bd390b 100644 --- a/guardian-service/src/api/tool.service.ts +++ b/guardian-service/src/api/tool.service.ts @@ -1,6 +1,9 @@ import { ApiResponse } from '../api/helpers/api-response.js'; import { BinaryMessageResponse, DatabaseServer, Hashing, INotificationStep, MessageAction, MessageError, MessageResponse, MessageServer, MessageType, NewNotifier, PinoLogger, Policy, PolicyTool, replaceAllEntities, replaceAllVariables, RunFunctionAsync, SchemaFields, ToolImportExport, ToolMessage, TopicConfig, TopicHelper, Users } from '@guardian/common'; -import { IOwner, IRootConfig, MessageAPI, ModelHelper, ModuleStatus, PolicyStatus, SchemaStatus, TagType, TopicType } from '@guardian/interfaces'; +import { + GenerateUUIDv4, + IOwner, IRootConfig, MessageAPI, ModelHelper, ModuleStatus, PolicyStatus, SchemaStatus, TagType, TopicType +} from '@guardian/interfaces'; import { ISerializedErrors } from '../policy-engine/policy-validation-results-container.js'; import { ToolValidator } from '../policy-engine/block-validators/tool-validator.js'; import { PolicyConverterUtils } from '../helpers/import-helpers/policy/policy-converter-utils.js'; @@ -203,6 +206,7 @@ export async function publishTool( const STEP_PUBLISH_SCHEMAS = 'Publish schemas'; const STEP_CREATE_TAGS_TOPIC = 'Create tags topic'; const STEP_GENERATE_FILE = 'Generate file'; + const STEP_SAVE_FILE_IN_DB = 'Save file in database'; const STEP_PUBLISH_TOOL = 'Publish tool'; const STEP_PUBLISH_TAGS = 'Publish tags'; const STEP_SAVE = 'Save'; @@ -213,6 +217,7 @@ export async function publishTool( notifier.addStep(STEP_PUBLISH_SCHEMAS); notifier.addStep(STEP_CREATE_TAGS_TOPIC); notifier.addStep(STEP_GENERATE_FILE); + notifier.addStep(STEP_SAVE_FILE_IN_DB); notifier.addStep(STEP_PUBLISH_TOOL); notifier.addStep(STEP_PUBLISH_TAGS); notifier.addStep(STEP_SAVE); @@ -269,11 +274,16 @@ export async function publishTool( compression: 'DEFLATE', compressionOptions: { level: 3 - } + }, + platform: 'UNIX', }); tool.hash = sha256(buffer); notifier.completeStep(STEP_GENERATE_FILE); + notifier.startStep(STEP_SAVE_FILE_IN_DB); + tool.contentFileId = await DatabaseServer.saveFile(GenerateUUIDv4(), Buffer.from(buffer)); + notifier.completeStep(STEP_SAVE_FILE_IN_DB); + notifier.startStep(STEP_PUBLISH_TOOL); const message = new ToolMessage(MessageType.Tool, MessageAction.PublishTool); message.setDocument(tool, buffer); @@ -546,7 +556,8 @@ export async function dryRunTool( compression: 'DEFLATE', compressionOptions: { level: 3 - } + }, + platform: 'UNIX', }); tool.hash = sha256(buffer); @@ -953,6 +964,13 @@ export async function toolsAPI(logger: PinoLogger): Promise { throw new Error('Invalid tool'); } + if (item.status === ModuleStatus.PUBLISHED && item.contentFileId) { + const buffer = await DatabaseServer.loadFile(item.contentFileId); + const arrayBuffer = Uint8Array.from(buffer).buffer + + return new BinaryMessageResponse(arrayBuffer); + } + await updateToolConfig(item); const zip = await ToolImportExport.generate(item); const file = await zip.generateAsync({ @@ -961,6 +979,7 @@ export async function toolsAPI(logger: PinoLogger): Promise { compressionOptions: { level: 3, }, + platform: 'UNIX' }); return new BinaryMessageResponse(file); } catch (error) { diff --git a/guardian-service/src/helpers/import-helpers/schema/schema-publish-helper.ts b/guardian-service/src/helpers/import-helpers/schema/schema-publish-helper.ts index efe04d9e77..c37d4ff1d9 100644 --- a/guardian-service/src/helpers/import-helpers/schema/schema-publish-helper.ts +++ b/guardian-service/src/helpers/import-helpers/schema/schema-publish-helper.ts @@ -1,4 +1,15 @@ -import { GeoJsonContext, IOwner, IRootConfig, ISchemaDocument, ModuleStatus, Schema, SchemaHelper, SchemaStatus, SentinelHubContext } from '@guardian/interfaces'; +import { + GenerateUUIDv4, + GeoJsonContext, + IOwner, + IRootConfig, + ISchemaDocument, + ModuleStatus, + Schema, + SchemaHelper, + SchemaStatus, + SentinelHubContext +} from '@guardian/interfaces'; import { DatabaseServer, INotificationStep, MessageAction, MessageServer, Schema as SchemaCollection, SchemaMessage, SchemaPackageMessage, schemasToContext, TopicConfig, UrlType } from '@guardian/common'; import { checkForCircularDependency } from '../common/load-helper.js'; import { incrementSchemaVersion, updateSchemaDefs, updateSchemaDocument } from './schema-helper.js'; @@ -159,6 +170,12 @@ export async function publishSchema( const relationships = await SchemaImportExportHelper.exportSchemas([item.id]); + const documentBuffer = Buffer.from(JSON.stringify(item.document)); + const contextBuffer = Buffer.from(JSON.stringify(item.context)); + + item.contentDocumentFileId = (await DatabaseServer.saveFile(GenerateUUIDv4(), documentBuffer)).toString(); + item.contentContextFileId = (await DatabaseServer.saveFile(GenerateUUIDv4(), contextBuffer)).toString(); + const message = new SchemaMessage(type || MessageAction.PublishSchema); message.setDocument(item); message.setRelationships(relationships); diff --git a/guardian-service/src/helpers/import-helpers/tag/tag-publish-helper.ts b/guardian-service/src/helpers/import-helpers/tag/tag-publish-helper.ts index d136b52d2b..25e309e375 100644 --- a/guardian-service/src/helpers/import-helpers/tag/tag-publish-helper.ts +++ b/guardian-service/src/helpers/import-helpers/tag/tag-publish-helper.ts @@ -12,7 +12,7 @@ import { TopicConfig, UrlType } from '@guardian/common'; -import { IOwner, IRootConfig, TagType } from '@guardian/interfaces'; +import {GenerateUUIDv4, IOwner, IRootConfig, TagType} from '@guardian/interfaces'; // /** // * Publish schema tags @@ -250,6 +250,10 @@ export async function publishTag( item.date = item.date || (new Date()).toISOString(); const message = new TagMessage(MessageAction.PublishTag); message.setDocument(item); + + const buffer = Buffer.from(JSON.stringify(item.document)); + item.contentFileId = (await DatabaseServer.saveFile(GenerateUUIDv4(), buffer)).toString(); + const result = await messageServer .sendMessage(message, { sendToIPFS: true, @@ -263,4 +267,4 @@ export async function publishTag( item.topicId = topicId; item.uri = result.getDocumentUrl(UrlType.url); return item; -} \ No newline at end of file +} diff --git a/guardian-service/src/helpers/import-helpers/tool/tool-import-helper.ts b/guardian-service/src/helpers/import-helpers/tool/tool-import-helper.ts index fc46fdd4cb..699e95e883 100644 --- a/guardian-service/src/helpers/import-helpers/tool/tool-import-helper.ts +++ b/guardian-service/src/helpers/import-helpers/tool/tool-import-helper.ts @@ -82,12 +82,14 @@ export async function importToolByMessage( ): Promise { // <-- Steps const STEP_LOAD_FILE = 'Load tool file'; + const STEP_SAVE_FILE_IN_DB = 'Save file in database'; const STEP_PARSE_FILE = 'Parse tool file'; const STEP_IMPORT_SCHEMAS = 'Import tool schemas'; const STEP_IMPORT_TAGS = 'Import tool tags'; // Steps --> notifier.addStep(STEP_LOAD_FILE); + notifier.addStep(STEP_SAVE_FILE_IN_DB); notifier.addStep(STEP_PARSE_FILE); notifier.addStep(STEP_IMPORT_SCHEMAS); notifier.addStep(STEP_IMPORT_TAGS); @@ -123,6 +125,7 @@ export async function importToolByMessage( if (!message.document) { throw new Error('File in body is empty'); } + const oldTool = await DatabaseServer.getTool({ messageId }); if (oldTool) { if ( @@ -157,6 +160,11 @@ export async function importToolByMessage( } notifier.completeStep(STEP_LOAD_FILE); + notifier.startStep(STEP_SAVE_FILE_IN_DB); + const buffer = Buffer.from(message.document); + const contentFileId = await DatabaseServer.saveFile(GenerateUUIDv4(), buffer); + notifier.completeStep(STEP_SAVE_FILE_IN_DB); + notifier.startStep(STEP_PARSE_FILE); const components = await ToolImportExport.parseZipFile(message.document); @@ -175,6 +183,7 @@ export async function importToolByMessage( components.tool.status = ModuleStatus.PUBLISHED; await updateToolConfig(components.tool); + components.tool.contentFileId = contentFileId; const result = await DatabaseServer.createTool(components.tool); notifier.completeStep(STEP_PARSE_FILE); diff --git a/guardian-service/src/policy-engine/policy-engine.service.ts b/guardian-service/src/policy-engine/policy-engine.service.ts index d58fb69c35..d8db3ff0e2 100644 --- a/guardian-service/src/policy-engine/policy-engine.service.ts +++ b/guardian-service/src/policy-engine/policy-engine.service.ts @@ -23,7 +23,7 @@ import { NotificationStep, PinoLogger, Policy, - PolicyAction, + PolicyAction, PolicyComment, PolicyDiscussion, PolicyImportExport, PolicyMessage, @@ -1453,6 +1453,14 @@ export class PolicyEngineService { const { policyId, owner } = msg; const policy = await DatabaseServer.getPolicyById(policyId); await this.policyEngine.accessPolicy(policy, owner, 'read'); + + if (policy.status === PolicyStatus.PUBLISH && policy.contentFileId) { + const buffer = await DatabaseServer.loadFile(policy.contentFileId); + const arrayBuffer = Uint8Array.from(buffer).buffer; + + return new BinaryMessageResponse(arrayBuffer); + } + const zip = await PolicyImportExport.generate(policy); const file = await zip.generateAsync({ type: 'arraybuffer', @@ -3922,7 +3930,7 @@ export class PolicyEngineService { cid: string; }[]; }, - }): Promise> => { + }): Promise> => { try { const { user, documentId, policyId, discussionId, data } = msg; diff --git a/guardian-service/src/policy-engine/policy-engine.ts b/guardian-service/src/policy-engine/policy-engine.ts index e630b3e798..ff2939f7aa 100644 --- a/guardian-service/src/policy-engine/policy-engine.ts +++ b/guardian-service/src/policy-engine/policy-engine.ts @@ -1000,6 +1000,7 @@ export class PolicyEngine extends NatsService { const STEP_CREATE_RESTORE_TOPIC = 'Create restore topic'; const STEP_CREATE_ACTION_TOPIC = 'Create actions topic'; const STEP_CREATE_COMMENTS_TOPIC = 'Create comments topic'; + const STEP_SAVE_FILE_IN_DB = 'Save file in database'; const STEP_PUBLISH_POLICY = 'Publish policy'; const STEP_PUBLISH_MESSAGE = 'Publish message'; const STEP_PUBLISH_TAGS = 'Publish tags'; @@ -1016,7 +1017,8 @@ export class PolicyEngine extends NatsService { notifier.addStep(STEP_CREATE_RESTORE_TOPIC, 2); notifier.addStep(STEP_CREATE_ACTION_TOPIC, 2); notifier.addStep(STEP_CREATE_COMMENTS_TOPIC, 2); - notifier.addStep(STEP_PUBLISH_POLICY, 20); + notifier.addStep(STEP_SAVE_FILE_IN_DB, 2); + notifier.addStep(STEP_PUBLISH_POLICY, 18); notifier.addStep(STEP_PUBLISH_MESSAGE, 4); notifier.addStep(STEP_PUBLISH_TAGS, 4); notifier.addStep(STEP_SAVE, 2); @@ -1304,6 +1306,10 @@ export class PolicyEngine extends NatsService { } }); + notifier.startStep(STEP_SAVE_FILE_IN_DB); + model.contentFileId = await DatabaseServer.saveFile(GenerateUUIDv4(), Buffer.from(buffer)); + notifier.completeStep(STEP_SAVE_FILE_IN_DB); + notifier.startStep(STEP_PUBLISH_POLICY); let currentHash; diff --git a/interfaces/src/interface/schema.interface.ts b/interfaces/src/interface/schema.interface.ts index 9967d0aeb6..c45d348264 100644 --- a/interfaces/src/interface/schema.interface.ts +++ b/interfaces/src/interface/schema.interface.ts @@ -119,4 +119,14 @@ export interface ISchema { * Errors */ errors?: any[]; + + /** + * Document file id of the original schema(publish flow). + */ + contentDocumentFileId?: string; + + /** + * Context file id of the original schema(publish flow). + */ + contentContextFileId?: string; }