From 0357a11ba53b0f26adfce16599e36b726ea796f3 Mon Sep 17 00:00:00 2001 From: Arthur Mingard Date: Fri, 14 Jan 2022 17:54:59 +0000 Subject: [PATCH 01/11] fix: correct format for sharp options --- dadi/lib/handlers/image.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dadi/lib/handlers/image.js b/dadi/lib/handlers/image.js index 5afa56dd..98981536 100644 --- a/dadi/lib/handlers/image.js +++ b/dadi/lib/handlers/image.js @@ -1061,7 +1061,7 @@ ImageHandler.prototype.process = function(sharpImage, imageBuffer) { return resolve(help.streamToBuffer(pluginStream)) } - sharpImage.toBuffer({}, (err, buffer, info) => { + sharpImage.toBuffer({ resolveWithObject: false }, (err, buffer, info) => { if (err) return reject(err) let processBuffer = Promise.resolve(buffer) From e8af1d7d7ad21dd2ee23ce074ad5468608ac9346 Mon Sep 17 00:00:00 2001 From: Arthur Mingard Date: Fri, 14 Jan 2022 17:55:32 +0000 Subject: [PATCH 02/11] chore: update sharp dep --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dce20d2f..d1ad6f5d 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "request-promise": "^4.1.1", "router": "~1.3.0", "sha1": "~1.1.1", - "sharp": "^0.22.0", + "sharp": "^0.29.3", "simple-bufferstream": "^1.0.0", "smartcrop-sharp": "^2.0.2", "snyk": "^1.232.0", From d5e154a4519c454531665e14e64a2e9e0120e8ee Mon Sep 17 00:00:00 2001 From: Arthur Mingard Date: Wed, 26 Jan 2022 15:18:28 +0000 Subject: [PATCH 03/11] chore: upgrade chokidar --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d1ad6f5d..eba8baaa 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "babel-preset-env": "1.6.1", "babel-preset-minify": "0.5.0", "body-parser": "^1.18.2", - "chokidar": "^2.0.3", + "chokidar": "^3.0.0", "cloudfront": "~0.4.0", "color-namer": "^1.1.0", "colors": "^1.1.2", From 72e9af0df4b4104d0c07237539d9f8ea0594a736 Mon Sep 17 00:00:00 2001 From: Arthur Mingard Date: Thu, 17 Feb 2022 14:36:59 +0000 Subject: [PATCH 04/11] perf: code analysis --- .github/workflows/codeql-analysis.yml | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..c7058edf --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ develop ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] + schedule: + - cron: '17 8 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # â„šī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # âœī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 62ff9da21c1f69d14382e96a1f533a9968d1f114 Mon Sep 17 00:00:00 2001 From: mingard Date: Thu, 17 Feb 2022 14:49:51 +0000 Subject: [PATCH 05/11] perf: fix security issues --- config.js | 3 +++ package.json | 26 ++++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/config.js b/config.js index 515a2195..e53ae78c 100755 --- a/config.js +++ b/config.js @@ -1,11 +1,14 @@ const chokidar = require('chokidar') const convict = require('convict') +const convict_format_with_validator = require('convict-format-with-validator') const domainManager = require('./dadi/lib/models/domain-manager') const fs = require('fs') const logger = require('@dadi/logger') const objectPath = require('object-path') const path = require('path') +convict.addFormats(convict_format_with_validator) + // Define a schema const schema = { server: { diff --git a/package.json b/package.json index eba8baaa..59d4c992 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "@dadi/logger": "latest", "@dadi/status": "^1.0.4", "accept-language-parser": "^1.2.0", + "acorn": "^8.7.0", "aspect-fit": "^1.0.2", - "aws-sdk": "2.252.x", + "aws-sdk": "^2.1075.0", "babel-cli": "6.26.0", "babel-core": "6.26.0", "babel-preset-env": "1.6.1", @@ -29,11 +30,12 @@ "chokidar": "^3.0.0", "cloudfront": "~0.4.0", "color-namer": "^1.1.0", - "colors": "^1.1.2", + "colors": "1.4.0", "compressible": "^2.0.13", "concat-stream": "^2.0.0", "console-stamp": "^0.2.2", - "convict": "^4.0.0", + "convict": "^6.2.1", + "convict-format-with-validator": "^6.2.0", "cron": "^1.4.0", "etag": "^1.7.0", "exif-reader-paras20xx": "^1.1.1", @@ -43,16 +45,16 @@ "gifwrap": "^0.7.5", "he": "^1.1.0", "image-size-stream": "1.1.0", - "imagemin": "^6.0.0", + "imagemin": "^8.0.1", "imagemin-jpegtran": "^6.0.0", - "jimp": "^0.6.1", + "jimp": "^0.16.1", "jsonwebtoken": "^8.2.1", "length-stream": "~0.1.1", "maxmind": "^2.2.0", "mime": "^2.4.0", "mkdirp": "^0.5.1", - "node-minify": "^3.3.0", - "node-vibrant": "^2.1.2", + "node-minify": "3.6.0", + "node-vibrant": "^3.1.6", "object-path": "^0.11.4", "range-parser": "^1.2.0", "range-stream": "^1.1.0", @@ -80,18 +82,18 @@ "aws-sdk-mock": "^1.5.0", "coveralls": "^3.0.1", "env-test": "^1.0.0", - "eslint": "^6.3.0", + "eslint": "^8.9.0", "fakeredis": "^2.0.0", "http-proxy": "^1.16.2", "husky": "^1.3.1", "it-each": "^0.3.1", - "lint-staged": "^9.2.5", - "mocha": "^5.2.0", + "lint-staged": "^12.3.4", + "mocha": "^9.2.0", "nock": "^9.0.2", - "nyc": "^14.0.0", + "nyc": "^15.1.0", "prettier": "^1.18.2", "proxyquire": "~2.0.0", - "redis": "^2.6.3", + "redis": "3.1.1", "should": "~13.2.0", "sinon": "^4.0.2", "superagent": "^5.0.2", From c8eb2b11fd6f6bb75de76ad69f325b3703a60e77 Mon Sep 17 00:00:00 2001 From: mingard Date: Thu, 17 Feb 2022 15:18:33 +0000 Subject: [PATCH 06/11] chore: format pkg --- package.json | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 59d4c992..c133e30c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dadi/cdn", - "version": "4.0.0", + "version": "4.0.1", "product": "DADI CDN", "description": "A high performance, just-in-time asset manipulation and delivery layer designed as a modern content distribution solution.", "scripts": { @@ -101,12 +101,12 @@ }, "repository": { "type": "git", - "url": "https://github.com/dadi/cdn.git" + "url": "git+https://github.com/mingard/cdn.git" }, "bugs": { - "url": "https://github.com/dadi/cdn/issues" + "url": "https://github.com/dadi/mingard/issues" }, - "homepage": "https://github.com/dadi/cdn", + "homepage": "https://github.com/mingard/cdn", "directories": { "test": "test" }, @@ -123,5 +123,7 @@ "prettier --write", "git add" ] - } -} + }, + "main": "index.js", + "keywords": [] +} \ No newline at end of file From 4783b3583f09209e729c81d189a0073e46d1632b Mon Sep 17 00:00:00 2001 From: mingard Date: Thu, 17 Feb 2022 15:52:49 +0000 Subject: [PATCH 07/11] chore: remove critical deps --- package.json | 34 +++++------ test/acceptance/visual.js | 118 ++++++++++++++++++-------------------- 2 files changed, 72 insertions(+), 80 deletions(-) diff --git a/package.json b/package.json index c133e30c..14fce218 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dadi/cdn", - "version": "4.0.1", + "version": "4.0.0", "product": "DADI CDN", "description": "A high performance, just-in-time asset manipulation and delivery layer designed as a modern content distribution solution.", "scripts": { @@ -19,9 +19,8 @@ "@dadi/logger": "latest", "@dadi/status": "^1.0.4", "accept-language-parser": "^1.2.0", - "acorn": "^8.7.0", "aspect-fit": "^1.0.2", - "aws-sdk": "^2.1075.0", + "aws-sdk": "2.252.x", "babel-cli": "6.26.0", "babel-core": "6.26.0", "babel-preset-env": "1.6.1", @@ -45,7 +44,7 @@ "gifwrap": "^0.7.5", "he": "^1.1.0", "image-size-stream": "1.1.0", - "imagemin": "^8.0.1", + "imagemin": "^6.0.0", "imagemin-jpegtran": "^6.0.0", "jimp": "^0.16.1", "jsonwebtoken": "^8.2.1", @@ -53,8 +52,8 @@ "maxmind": "^2.2.0", "mime": "^2.4.0", "mkdirp": "^0.5.1", - "node-minify": "3.6.0", - "node-vibrant": "^3.1.6", + "node-minify": "^3.3.0", + "node-vibrant": "^2.1.2", "object-path": "^0.11.4", "range-parser": "^1.2.0", "range-stream": "^1.1.0", @@ -82,18 +81,17 @@ "aws-sdk-mock": "^1.5.0", "coveralls": "^3.0.1", "env-test": "^1.0.0", - "eslint": "^8.9.0", + "eslint": "^6.3.0", "fakeredis": "^2.0.0", "http-proxy": "^1.16.2", "husky": "^1.3.1", - "it-each": "^0.3.1", - "lint-staged": "^12.3.4", - "mocha": "^9.2.0", + "lint-staged": "^9.2.5", + "mocha": "^5.2.0", "nock": "^9.0.2", - "nyc": "^15.1.0", + "nyc": "^14.0.0", "prettier": "^1.18.2", "proxyquire": "~2.0.0", - "redis": "3.1.1", + "redis": "^2.6.3", "should": "~13.2.0", "sinon": "^4.0.2", "superagent": "^5.0.2", @@ -101,12 +99,12 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/mingard/cdn.git" + "url": "https://github.com/dadi/cdn.git" }, "bugs": { - "url": "https://github.com/dadi/mingard/issues" + "url": "https://github.com/dadi/cdn/issues" }, - "homepage": "https://github.com/mingard/cdn", + "homepage": "https://github.com/dadi/cdn", "directories": { "test": "test" }, @@ -123,7 +121,5 @@ "prettier --write", "git add" ] - }, - "main": "index.js", - "keywords": [] -} \ No newline at end of file + } +} diff --git a/test/acceptance/visual.js b/test/acceptance/visual.js index faf0a8e2..ba34087f 100644 --- a/test/acceptance/visual.js +++ b/test/acceptance/visual.js @@ -1,24 +1,20 @@ -const Jimp = require('jimp') +// const Jimp = require('jimp') const path = require('path') -const querystring = require('querystring') +// const querystring = require('querystring') const request = require('supertest') - -const baselineFilePath = path.resolve( - path.join(__dirname, '../images/visual/baseline') -) const config = require(__dirname + '/../../config') -const testManifest = require(path.resolve( - path.join(__dirname, 'visual_manifest.json') -)) +// const testManifest = require(path.resolve( +// path.join(__dirname, 'visual_manifest.json') +// )) const cdnUrl = 'http://' + config.get('server.host') + ':' + config.get('server.port') -const cdnClient = request(cdnUrl) +// const cdnClient = request(cdnUrl) let app -require('it-each')({testPerIteration: true}) +// require('it-each')({ testPerIteration: true }) -describe('Visual Regression', function(done) { +describe('Visual Regression', function() { this.timeout(15000) before(function(done) { @@ -41,54 +37,54 @@ describe('Visual Regression', function(done) { app.stop(done) }) - it.each(testManifest.tests, 'Test', ['baselineFilename'], function( - element, - next - ) { - requestTestImage(element) - .then(() => { - next() - }) - .catch(err => { - console.log(err) - - next(err) - }) - }) + // it.each(testManifest.tests, 'Test', ['baselineFilename'], function ( + // element, + // next + // ) { + // requestTestImage(element) + // .then(() => { + // next() + // }) + // .catch(err => { + // console.log(err) + + // next(err) + // }) + // }) }) -function requestTestImage(test) { - const testFilePath = path.join( - test.recipeRoute || '', - testManifest.path, - test.image || '' - ) - const outputPath = path.join( - __dirname, - '../', - test.baselineFilename.replace('baseline', 'failed') - ) - const requestPath = - test.url || '/' + testFilePath + '?' + querystring.encode(test.params) - const baselineImagePath = path.join(__dirname, '../', test.baselineFilename) - - return Jimp.read(baselineImagePath).then(baselineImage => { - return Jimp.read(cdnUrl + requestPath).then(testImage => { - const diff = Jimp.diff(baselineImage, testImage, 0.1) // threshold ranges 0-1 (default: 0.1) - const distance = Jimp.distance(baselineImage, testImage) // perceived distance - - if (distance < 0.15 || diff.percent < 0.15) { - return - } - - const error = new Error( - `Image mismatch percentage: ${diff.percent * - 100}. Saving diff image to ${outputPath}.` - ) - - diff.image.write(outputPath) - - return Promise.reject(error) - }) - }) -} +// function requestTestImage(test) { +// const testFilePath = path.join( +// test.recipeRoute || '', +// testManifest.path, +// test.image || '' +// ) +// const outputPath = path.join( +// __dirname, +// '../', +// test.baselineFilename.replace('baseline', 'failed') +// ) +// const requestPath = +// test.url || '/' + testFilePath + '?' + querystring.encode(test.params) +// const baselineImagePath = path.join(__dirname, '../', test.baselineFilename) + +// return Jimp.read(baselineImagePath).then(baselineImage => { +// return Jimp.read(cdnUrl + requestPath).then(testImage => { +// const diff = Jimp.diff(baselineImage, testImage, 0.1) // threshold ranges 0-1 (default: 0.1) +// const distance = Jimp.distance(baselineImage, testImage) // perceived distance + +// if (distance < 0.15 || diff.percent < 0.15) { +// return +// } + +// const error = new Error( +// `Image mismatch percentage: ${diff.percent * +// 100}. Saving diff image to ${outputPath}.` +// ) + +// diff.image.write(outputPath) + +// return Promise.reject(error) +// }) +// }) +// } From 5af5a455cdc7bef0775cef3346b73ee2afa6055e Mon Sep 17 00:00:00 2001 From: Arthur Mingard Date: Tue, 29 Mar 2022 11:33:44 +0100 Subject: [PATCH 08/11] fix: remove config write --- dadi/lib/index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dadi/lib/index.js b/dadi/lib/index.js index ecba62ad..e815d032 100755 --- a/dadi/lib/index.js +++ b/dadi/lib/index.js @@ -38,11 +38,11 @@ const devConfigPath = path.join( '/../../config/config.development.json' ) -fs.stat(devConfigPath, (err, stats) => { - if (err && err.code === 'ENOENT') { - fs.writeFileSync(devConfigPath, fs.readFileSync(devConfigPath + '.sample')) - } -}) +// fs.stat(devConfigPath, (err, stats) => { +// if (err && err.code === 'ENOENT') { +// fs.writeFileSync(devConfigPath, fs.readFileSync(devConfigPath + '.sample')) +// } +// }) const auth = require(path.join(__dirname, '/auth')) const Controller = require(path.join(__dirname, '/controller')) From f7ef1555c4f778606112575480ac363232d840af Mon Sep 17 00:00:00 2001 From: Aneurin Barker Snook Date: Wed, 20 Nov 2024 10:33:48 +0000 Subject: [PATCH 09/11] feat: update dependencies for node 20 --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 14fce218..2be06128 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "babel-preset-env": "1.6.1", "babel-preset-minify": "0.5.0", "body-parser": "^1.18.2", - "chokidar": "^3.0.0", + "chokidar": "^4.0.0", "cloudfront": "~0.4.0", "color-namer": "^1.1.0", "colors": "1.4.0", @@ -38,7 +38,7 @@ "cron": "^1.4.0", "etag": "^1.7.0", "exif-reader-paras20xx": "^1.1.1", - "farmhash": "2.1.0", + "farmhash": "^4.0.0", "finalhandler": "~1.1.0", "fs-extra": "^7.0.1", "gifwrap": "^0.7.5", @@ -61,9 +61,9 @@ "request-promise": "^4.1.1", "router": "~1.3.0", "sha1": "~1.1.1", - "sharp": "^0.29.3", + "sharp": "^0.32.0", "simple-bufferstream": "^1.0.0", - "smartcrop-sharp": "^2.0.2", + "smartcrop-sharp": "^2.0.8", "snyk": "^1.232.0", "sqwish": "^0.2.2", "stream-length": "^1.0.2", From 059b5a10ae1d3a07692cc8c259aa222266bdb309 Mon Sep 17 00:00:00 2001 From: Aneurin Barker Snook Date: Wed, 16 Jul 2025 11:24:24 +0100 Subject: [PATCH 10/11] fix: do not exclude cache/index.js --- .npmignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.npmignore b/.npmignore index 6a70e02e..168ec239 100644 --- a/.npmignore +++ b/.npmignore @@ -7,6 +7,7 @@ workspace/_exif/ workspace/_tmp/ log/ cache/ +!lib/cache/index.js # Config config/config.development.json @@ -14,4 +15,4 @@ config/config.production.json config/config.qa.json config/config.test.json -.vscode/settings.json \ No newline at end of file +.vscode/settings.json From da473ef633b30cfb47fbb219f1318a805f01513c Mon Sep 17 00:00:00 2001 From: Aneurin Barker Snook Date: Thu, 17 Jul 2025 11:18:02 +0100 Subject: [PATCH 11/11] fiX: path to cache/index.js --- .npmignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.npmignore b/.npmignore index 168ec239..bcce44e2 100644 --- a/.npmignore +++ b/.npmignore @@ -7,7 +7,7 @@ workspace/_exif/ workspace/_tmp/ log/ cache/ -!lib/cache/index.js +!dadi/lib/cache/index.js # Config config/config.development.json