[WIP]feat: 4931 deterministic compression

api-gateway/src/api/service/schema.ts

@@ -1925,13 +1925,21 @@ export class SchemaApi {
             const ids = schemas.map(s => s.id);
             const tags = await guardians.exportTags(owner, 'Schema', ids);
             const name = `${Date.now()}`;
-            const zip = await SchemaImportExport.generateZipFile({ schemas, tags });
+
+            const zip = await SchemaImportExport.generateZipFile({
+                schemas,
+                tags,
+                helpers: guardians,
+                user
+            });
+
             const arcStream = zip.generateNodeStream({
                 type: 'nodebuffer',
                 compression: 'DEFLATE',
                 compressionOptions: {
                     level: 3
-                }
+                },
+                platform: 'UNIX',
             });
             res.header('Content-disposition', `attachment; filename=${name}`);
             res.header('Content-type', 'application/zip');

common/src/entity/formula.ts

@@ -1,6 +1,8 @@
-import { BeforeCreate, Entity, Property } from '@mikro-orm/core';
+import {AfterDelete, BeforeCreate, Entity, Property} from '@mikro-orm/core';
 import { BaseEntity } from '../models/index.js';
 import { EntityStatus, GenerateUUIDv4, IFormula, IFormulaConfig } from '@guardian/interfaces';
+import {ObjectId} from "@mikro-orm/mongodb";
+import {DataBaseHelper} from "../helpers/index.js";
 
 /**
  * Formula collection
@@ -85,6 +87,12 @@ export class Formula extends BaseEntity implements IFormula {
     @Property({ nullable: true, type: 'unknown' })
     config?: IFormulaConfig;
 
+    /**
+     * File id of the original formula zip (publish flow).
+     */
+    @Property({ nullable: true })
+    contentFileId?: ObjectId;
+
     /**
      * Set defaults
      */
@@ -93,4 +101,16 @@ export class Formula extends BaseEntity implements IFormula {
         this.uuid = this.uuid || GenerateUUIDv4();
         this.status = this.status || EntityStatus.DRAFT;
     }
-}
+
+    @AfterDelete()
+    deleteContentFile() {
+        if (this.contentFileId) {
+            DataBaseHelper.gridFS
+                .delete(this.contentFileId)
+                .catch((reason) => {
+                    console.error('AfterDelete: Formula, contentFileId');
+                    console.error(reason);
+                });
+        }
+    }
+}

common/src/entity/module.ts

@@ -94,6 +94,12 @@ export class PolicyModule extends BaseEntity {
     @Property({ persist: false, nullable: true })
     _configFileId?: ObjectId;
 
+    /**
+     * File id of the original module zip (publish flow).
+     */
+    @Property({ nullable: true })
+    contentFileId?: ObjectId;
+
     /**
      * Set defaults
      */
@@ -169,4 +175,19 @@ export class PolicyModule extends BaseEntity {
                 });
         }
     }
+
+    /**
+     * Delete original module zip (publish flow)
+     */
+    @AfterDelete()
+    deleteContentFile() {
+        if (this.contentFileId) {
+            DataBaseHelper.gridFS
+                .delete(this.contentFileId)
+                .catch((reason) => {
+                    console.error('AfterDelete: PolicyModule, contentFileId');
+                    console.error(reason);
+                });
+        }
+    }
 }

common/src/entity/policy-label.ts

@@ -1,6 +1,8 @@
-import { BeforeCreate, Entity, Property } from '@mikro-orm/core';
+import {AfterDelete, BeforeCreate, Entity, Property} from '@mikro-orm/core';
 import { BaseEntity } from '../models/index.js';
 import { EntityStatus, GenerateUUIDv4, IPolicyLabel, IPolicyLabelConfig } from '@guardian/interfaces';
+import { DataBaseHelper } from '../helpers/db-helper.js';
+import { ObjectId } from "@mikro-orm/mongodb";
 
 /**
  * PolicyLabel collection
@@ -100,6 +102,12 @@ export class PolicyLabel extends BaseEntity implements IPolicyLabel {
     @Property({ nullable: true })
     method?: string;
 
+    /**
+     * File id of the original Policy Label zip (publish flow).
+     */
+    @Property({ nullable: true })
+    contentFileId?: ObjectId;
+
     /**
      * Set defaults
      */
@@ -108,4 +116,19 @@ export class PolicyLabel extends BaseEntity implements IPolicyLabel {
         this.uuid = this.uuid || GenerateUUIDv4();
         this.status = this.status || EntityStatus.DRAFT;
     }
+
+    /**
+     * Delete original Policy Label zip (publish flow)
+     */
+    @AfterDelete()
+    deleteContentFile() {
+        if (this.contentFileId) {
+            DataBaseHelper.gridFS
+                .delete(this.contentFileId)
+                .catch((reason) => {
+                    console.error('AfterDelete: PolicyLabel, contentFileId');
+                    console.error(reason);
+                });
+        }
+    }
 }

common/src/entity/policy-statistic.ts

@@ -1,6 +1,8 @@
-import { BeforeCreate, Entity, Property } from '@mikro-orm/core';
+import {AfterDelete, BeforeCreate, Entity, Property} from '@mikro-orm/core';
 import { BaseEntity } from '../models/index.js';
 import { EntityStatus, GenerateUUIDv4, IStatistic, IStatisticConfig } from '@guardian/interfaces';
+import {ObjectId} from "@mikro-orm/mongodb";
+import { DataBaseHelper } from '../helpers/index.js';
 
 /**
  * PolicyStatistic collection
@@ -100,6 +102,12 @@ export class PolicyStatistic extends BaseEntity implements IStatistic {
     @Property({ nullable: true })
     method?: string;
 
+    /**
+     * File id of the original Policy Statistic file (publish flow).
+     */
+    @Property({ nullable: true })
+    contentFileId?: ObjectId;
+
     /**
      * Set defaults
      */
@@ -108,4 +116,19 @@ export class PolicyStatistic extends BaseEntity implements IStatistic {
         this.uuid = this.uuid || GenerateUUIDv4();
         this.status = this.status || EntityStatus.DRAFT;
     }
+
+    /**
+     * Delete original Policy Statistic file (publish flow)
+     */
+    @AfterDelete()
+    deleteContentFile() {
+        if (this.contentFileId) {
+            DataBaseHelper.gridFS
+                .delete(this.contentFileId)
+                .catch((reason) => {
+                    console.error('AfterDelete: PolicyStatistic, contentFileId');
+                    console.error(reason);
+                });
+        }
+    }
 }

common/src/entity/policy.ts

@@ -296,6 +296,13 @@ export class Policy extends BaseEntity {
 
     @Property({ nullable: true })
     originalMessageId?: string;
+
+    /**
+     * File id of the original policy zip (publish flow).
+     */
+    @Property({ nullable: true })
+    contentFileId?: ObjectId;
+
     /**
      * Set policy defaults
      */
@@ -407,4 +414,19 @@ export class Policy extends BaseEntity {
                 });
         }
     }
+
+    /**
+     * Delete original policy zip (publish flow)
+     */
+    @AfterDelete()
+    deleteContentFile() {
+        if (this.contentFileId) {
+            DataBaseHelper.gridFS
+                .delete(this.contentFileId)
+                .catch((reason) => {
+                    console.error('AfterDelete: Policy, contentFileId');
+                    console.error(reason);
+                });
+        }
+    }
 }

common/src/entity/schema.ts

@@ -193,6 +193,18 @@ export class Schema extends BaseEntity implements ISchema {
     @Property({ persist: false, nullable: true })
     _contextFileId?: ObjectId;
 
+    /**
+     * Document file id of the original schema(publish flow).
+     */
+    @Property({ nullable: true })
+    contentDocumentFileId?: string;
+
+    /**
+     * Context file id of the original schema(publish flow).
+     */
+    @Property({ nullable: true })
+    contentContextFileId?: string;
+
     /**
      * Schema defaults
      */
@@ -323,4 +335,27 @@ export class Schema extends BaseEntity implements ISchema {
                 });
         }
     }
+
+    /**
+     * Delete original schema document and context(publish flow)
+     */
+    @AfterDelete()
+    deleteContentFiles() {
+        if (this.contentDocumentFileId) {
+            DataBaseHelper.gridFS
+                .delete(new ObjectId(this.contentDocumentFileId))
+                .catch((reason) => {
+                    console.error('AfterDelete: Schema, contentDocumentFileId');
+                    console.error(reason);
+                });
+        }
+        if (this.contentContextFileId) {
+            DataBaseHelper.gridFS
+                .delete(new ObjectId(this.contentContextFileId))
+                .catch((reason) => {
+                    console.error('AfterDelete: Schema, contentContextFileId');
+                    console.error(reason);
+                });
+        }
+    }
 }

common/src/entity/tag.ts

@@ -3,6 +3,7 @@ import { RestoreEntity } from '../models/index.js';
 import { GenerateUUIDv4 } from '@guardian/interfaces';
 import { DataBaseHelper } from '../helpers/db-helper.js';
 import { DeleteCache } from './delete-cache.js';
+import { ObjectId } from "@mikro-orm/mongodb";
 
 /**
  * Tags collection
@@ -102,6 +103,12 @@ export class Tag extends RestoreEntity {
     @Property({ nullable: false })
     date: string;
 
+    /**
+     * File id of the original tag (publish flow).
+     */
+    @Property({ nullable: true })
+    contentFileId?: string;
+
     /**
      * Create document
      */
@@ -151,4 +158,19 @@ export class Tag extends RestoreEntity {
             console.error(error);
         }
     }
+
+    /**
+     * Delete original tag file (publish flow)
+     */
+    @AfterDelete()
+    deleteContentFile() {
+        if (this.contentFileId) {
+            DataBaseHelper.gridFS
+                .delete(new ObjectId(this.contentFileId))
+                .catch((reason) => {
+                    console.error('AfterDelete: Tag, contentFileId');
+                    console.error(reason);
+                });
+        }
+    }
 }

common/src/entity/tool.ts

@@ -111,6 +111,12 @@ export class PolicyTool extends BaseEntity {
     @Property({ persist: false, nullable: true })
     _configFileId?: ObjectId;
 
+    /**
+     * File id of the original tool zip (imported from IPFS or publish flow).
+     */
+    @Property({ nullable: true })
+    contentFileId?: ObjectId;
+
     /**
      * Set defaults
      */
@@ -186,4 +192,16 @@ export class PolicyTool extends BaseEntity {
                 });
         }
     }
+
+    @AfterDelete()
+    deleteContentFile() {
+        if (this.contentFileId) {
+            DataBaseHelper.gridFS
+                .delete(this.contentFileId)
+                .catch((reason) => {
+                    console.error('AfterDelete: PolicyTool, contentFileId');
+                    console.error(reason);
+                });
+        }
+    }
 }

common/src/import-export/formula.ts

@@ -2,6 +2,7 @@ import JSZip from 'jszip';
 import { Formula, Policy } from '../entity/index.js';
 import { IFormulaConfig } from '@guardian/interfaces';
 import { DatabaseServer } from '../database-modules/index.js';
+import { ImportExportUtils } from './utils.js';
 
 interface ISchemaComponents {
     iri: string;
@@ -90,8 +91,11 @@ export class FormulaImportExport {
         delete formulas.updateDate;
         const schemas = components.schemas;
         const zip = new JSZip();
-        zip.file(FormulaImportExport.schemasFileName, JSON.stringify(schemas));
-        zip.file(FormulaImportExport.formulaFileName, JSON.stringify(formulas));
+
+        const ZIP_FILE_OPTIONS = ImportExportUtils.getDeterministicZipFileOptions();
+
+        zip.file(FormulaImportExport.schemasFileName, JSON.stringify(schemas), ZIP_FILE_OPTIONS);
+        zip.file(FormulaImportExport.formulaFileName, JSON.stringify(formulas), ZIP_FILE_OPTIONS);
         return zip;
     }