🎨 Refactor the download system some

package.json

@@ -38,6 +38,7 @@
   "dependencies": {
     "axios": "^0.21.1",
     "chalk": "^4.1.0",
+    "cli-progress": "^3.9.1",
     "commander": "^6.2.1",
     "execa": "^5.1.1",
     "fs-extra": "^9.0.1",
@@ -49,6 +50,7 @@
     "sharp": "^0.29.1"
   },
   "devDependencies": {
+    "@types/cli-progress": "^3.9.2",
     "@types/fs-extra": "^9.0.6",
     "@types/node": "^14.14.16",
     "@types/prompts": "^2.0.14",

src/cmds.ts

@@ -103,8 +103,8 @@ export const commands: Cmd[] = [
     controller: importPatches,
   },
   {
-    cmd: 'init <source>',
+    cmd: 'ff-init <source>',
-    aliases: ['initialise', 'initialize'],
+    aliases: ['ff-initialise', 'ff-initialize'],
     description: 'Initialise the Firefox directory.',
     controller: init,
   },

src/commands/download-artifacts.ts

@@ -1,11 +1,10 @@
-import axios from 'axios'
 import execa from 'execa'
-import fs from 'fs'
 import { homedir } from 'os'
 import { posix, resolve, sep } from 'path'
 import { log } from '..'
+import { downloadFileToLocation } from '../utils/download'
 
-export const downloadArtifacts = async () => {
+export const downloadArtifacts = async (): Promise<void> => {
   if (process.platform !== 'win32')
     return log.error(
       'This is not a Windows machine, will not download artifacts.'
@@ -25,37 +24,11 @@ export const downloadArtifacts = async () => {
 
   log.info(`Downloading Windows artifacts...`)
 
-  const { data, headers } = await axios.get(url, {
+  await downloadFileToLocation(url, resolve(process.cwd(), filename))
-    responseType: 'stream',
-  })
 
-  const length = headers['content-length']
-
-  const writer = fs.createWriteStream(resolve(process.cwd(), filename))
-
-  let receivedBytes = 0
-
-  data.on('data', (chunk: any) => {
-    receivedBytes += chunk.length
-
-    const rand = Math.floor(Math.random() * 1000 + 1)
-
-    if (rand > 999.5) {
-      const percentCompleted = parseInt(
-        Math.round((receivedBytes * 100) / length).toFixed(0)
-      )
-      if (percentCompleted % 2 == 0 || percentCompleted >= 100) return
-      log.info(`\t${filename}\t${percentCompleted}%...`)
-    }
-  })
-
-  data.pipe(writer)
-
-  data.on('end', async () => {
   log.info('Unpacking mozbuild...')
 
   await execa('tar', ['-xvf', filename, '-C', home])
 
   log.info('Done extracting mozbuild artifacts.')
-  })
 }

src/commands/download.ts

@@ -1,14 +1,14 @@
-import axios from 'axios'
 import chalk from 'chalk'
 import execa from 'execa'
-import fs, { existsSync, rmdirSync, writeFileSync } from 'fs'
+import fs, { existsSync, rmdirSync } from 'fs'
 import { ensureDirSync, removeSync } from 'fs-extra'
 import ora from 'ora'
 import { homedir } from 'os'
 import { posix, resolve, sep } from 'path'
-import { bin_name, config, log } from '..'
+import { bin_name, log } from '..'
 import { ENGINE_DIR } from '../constants'
 import { getConfig, getLatestFF, writeMetadata } from '../utils'
+import { downloadFileToLocation } from '../utils/download'
 import { downloadArtifacts } from './download-artifacts'
 
 const gFFVersion = getConfig().version.version
@@ -23,6 +23,78 @@ let initProgress: any = ora({
   indent: 0,
 })
 
+export const download = async () => {
+  const version = gFFVersion
+
+  // If gFFVersion isn't specified, provide legible error
+  if (!version) {
+    log.error(
+      'You have not specified a version of firefox in your config file. This is required to build a firefox fork'
+    )
+    process.exit(1)
+  }
+
+  // The location to download the firefox source code from the web
+  const base = `https://archive.mozilla.org/pub/firefox/releases/${version}/source/`
+  const filename = `firefox-${version}.source.tar.xz`
+
+  const url = base + filename
+
+  log.info(`Locating Firefox release ${version}...`)
+
+  ensureDirSync(resolve(process.cwd(), `.dotbuild`, `engines`))
+
+  if (
+    existsSync(
+      resolve(
+        process.cwd(),
+        `.dotbuild`,
+        `engines`,
+        `firefox-${version.split('b')[0]}`
+      )
+    )
+  ) {
+    log.error(
+      `Cannot download version ${
+        version.split('b')[0]
+      } as it already exists at "${resolve(
+        process.cwd(),
+        `firefox-${version.split('b')[0]}`
+      )}"`
+    )
+  }
+
+  if (version.includes('b'))
+    log.warning(
+      'Version includes non-numeric characters. This is probably a beta.'
+    )
+
+  // Do not re-download if there is already an existing workspace present
+  if (existsSync(ENGINE_DIR)) {
+    log.error(
+      `Workspace already exists.\nRemove that workspace and run |${bin_name} download ${version}| again.`
+    )
+  }
+
+  log.info(`Downloading Firefox release ${version}...`)
+
+  await downloadFileToLocation(
+    url,
+    resolve(process.cwd(), `.dotbuild`, `engines`, filename)
+  )
+
+  await unpack(filename, version)
+
+  if (process.platform === 'win32') {
+    if (existsSync(resolve(homedir(), '.mozbuild'))) {
+      log.info('Mozbuild directory already exists, not redownloading')
+    } else {
+      log.info('Mozbuild not found, downloading artifacts.')
+      await downloadArtifacts()
+    }
+  }
+}
+
 const onData = (data: any) => {
   const d = data.toString()
 
@@ -72,7 +144,7 @@ const unpack = async (name: string, version: string) => {
   tarProc.on('exit', () => {
     if (process.env.CI_SKIP_INIT) return log.info('Skipping initialisation.')
 
-    const initProc = execa('npx', ['melon', 'init', 'engine'])
+    const initProc = execa('npx', ['melon', 'ff-init', 'engine'])
 
     ;(initProc.stdout as any).on('data', onData)
     ;(initProc.stdout as any).on('error', onData)
@@ -82,8 +154,6 @@ const unpack = async (name: string, version: string) => {
       initProgress.stop()
       initProgress = null
 
-      await new Promise((resolve) => setTimeout(resolve, 5000))
-
       log.success(
         `You should be ready to make changes to Dot Browser.\n\n\t   You should import the patches next, run |${bin_name} import|.\n\t   To begin building Dot, run |${bin_name} build|.`
       )
@@ -97,119 +167,3 @@ const unpack = async (name: string, version: string) => {
     })
   })
 }
-
-export const download = async (firefoxVersion?: string) => {
-  if (firefoxVersion)
-    log.warning(
-      `A custom Firefox version is being used. Some features of Dot may not work as expected.`
-    )
-
-  if (!firefoxVersion) {
-    firefoxVersion = gFFVersion
-  }
-
-  let version = await getLatestFF()
-
-  if (firefoxVersion) {
-    version = firefoxVersion
-  }
-
-  const base = `https://archive.mozilla.org/pub/firefox/releases/${version}/source/`
-  const filename = `firefox-${version}.source.tar.xz`
-
-  const url = `${base}${filename}`
-
-  log.info(`Locating Firefox release ${version}...`)
-
-  ensureDirSync(resolve(process.cwd(), `.dotbuild`, `engines`))
-
-  if (
-    existsSync(
-      resolve(
-        process.cwd(),
-        `.dotbuild`,
-        `engines`,
-        `firefox-${version.split('b')[0]}`
-      )
-    )
-  ) {
-    log.error(
-      `Cannot download version ${
-        version.split('b')[0]
-      } as it already exists at "${resolve(
-        process.cwd(),
-        `firefox-${version.split('b')[0]}`
-      )}"`
-    )
-  }
-
-  if (version == firefoxVersion)
-    log.info(`Version is frozen at ${firefoxVersion}!`)
-  if (version.includes('b'))
-    log.warning(
-      'Version includes non-numeric characters. This is probably a beta.'
-    )
-
-  if (
-    fs.existsSync(
-      resolve(
-        process.cwd(),
-        `.dotbuild`,
-        `engines`,
-        'firefox',
-        version.split('b')[0]
-      )
-    ) ||
-    fs.existsSync(
-      resolve(process.cwd(), 'firefox', `firefox-${  version.split('b')[0]}`)
-    )
-  )
-    log.error(
-      `Workspace with version "${
-        version.split('b')[0]
-      }" already exists.\nRemove that workspace and run |${bin_name} download ${version}| again.`
-    )
-
-  log.info(`Downloading Firefox release ${version}...`)
-
-  const { data, headers } = await axios.get(url, {
-    responseType: 'stream',
-  })
-
-  const length = headers['content-length']
-
-  const writer = fs.createWriteStream(
-    resolve(process.cwd(), `.dotbuild`, `engines`, filename)
-  )
-
-  let receivedBytes = 0
-
-  data.on('data', (chunk: any) => {
-    receivedBytes += chunk.length
-
-    const rand = Math.floor(Math.random() * 1000 + 1)
-
-    if (rand > 999.5) {
-      const percentCompleted = parseInt(
-        Math.round((receivedBytes * 100) / length).toFixed(0)
-      )
-      if (percentCompleted % 2 == 0 || percentCompleted >= 100) return
-      log.info(`\t${filename}\t${percentCompleted}%...`)
-    }
-  })
-
-  data.pipe(writer)
-
-  data.on('end', async () => {
-    await unpack(filename, version)
-
-    if (process.platform === 'win32') {
-      if (existsSync(resolve(homedir(), '.mozbuild'))) {
-        log.info('Mozbuild directory already exists, not redownloading')
-      } else {
-        log.info('Mozbuild not found, downloading artifacts.')
-        await downloadArtifacts()
-      }
-    }
-  })
-}

src/commands/init.ts

@@ -4,7 +4,7 @@ import { resolve } from 'path'
 import { bin_name, log } from '..'
 import { dispatch } from '../utils'
 
-export const init = async (directory: Command) => {
+export const init = async (directory: Command): Promise<void> => {
   if (process.platform == 'win32') {
     // Because Windows cannot handle paths correctly, we're just calling a script as the workaround.
     log.info(
@@ -41,9 +41,10 @@ export const init = async (directory: Command) => {
 
   version = version.trim().replace(/\\n/g, '')
 
+  log.info('Initializing git, this may take some time')
   await dispatch('git', ['init'], dir as string)
   await dispatch('git', ['checkout', '--orphan', version], dir as string)
-  await dispatch('git', ['add', '-v', '-f', '.'], dir as string)
+  await dispatch('git', ['add', '-f', '.'], dir as string)
   await dispatch(
     'git',
     ['commit', '-am', `"Firefox ${version}"`],

src/utils/download.ts

@@ -0,0 +1,43 @@
+import { createWriteStream } from 'fs'
+
+import axios from 'axios'
+import cliProgress from 'cli-progress'
+
+export async function downloadFileToLocation(
+  url: string,
+  writeOut: string
+): Promise<void> {
+  return new Promise((resolve, reject) =>
+    (async () => {
+      const { data, headers } = await axios.get(url, {
+        responseType: 'stream',
+      })
+
+      const length = headers['content-length']
+
+      const writer = createWriteStream(writeOut)
+
+      let receivedBytes = 0
+
+      const progressBar = new cliProgress.SingleBar({})
+      progressBar.start(length, receivedBytes)
+
+      data.on('data', (chunk: { length: number }) => {
+        receivedBytes += chunk.length
+      })
+      data.pipe(writer)
+      data.on('error', (err: unknown) => reject(err))
+
+      const progressInterval = setInterval(
+        () => progressBar.update(receivedBytes),
+        500
+      )
+
+      data.on('end', async () => {
+        clearInterval(progressInterval)
+        progressBar.stop()
+        resolve()
+      })
+    })()
+  )
+}

yarn.lock

@@ -120,6 +120,13 @@
   resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e"
   integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==
 
+"@types/cli-progress@^3.9.2":
+  version "3.9.2"
+  resolved "https://registry.yarnpkg.com/@types/cli-progress/-/cli-progress-3.9.2.tgz#6ca355f96268af39bee9f9307f0ac96145639c26"
+  integrity sha512-VO5/X5Ij+oVgEVjg5u0IXVe3JQSKJX+Ev8C5x+0hPy0AuWyW+bF8tbajR7cPFnDGhs7pidztcac+ccrDtk5teA==
+  dependencies:
+    "@types/node" "*"
+
 "@types/fs-extra@^9.0.6":
   version "9.0.12"
   resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-9.0.12.tgz#9b8f27973df8a7a3920e8461517ebf8a7d4fdfaf"
@@ -160,13 +167,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.17.17.tgz#4ec7b71bbcb01a4e55455b60b18b1b6a783fe31d"
   integrity sha512-niAjcewgEYvSPCZm3OaM9y6YQrL2SEPH9PymtE6fuZAvFiP6ereCcvApGl2jKTq7copTIguX3PBvfP08LN4LvQ==
 
-"@types/promptly@^3.0.2":
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/@types/promptly/-/promptly-3.0.2.tgz#598674d4b78b3dffcb2d756b344f28a2cf7459f8"
-  integrity sha512-cJFwE7d8GlraY+DJoZ0NhpoJ55slkcbNsGIKMY0H+5h0xaGqXBqXz9zeu+Ey9KfN1UiHQXiIT0GroxyPYMPP/w==
-  dependencies:
-    "@types/node" "*"
-
 "@types/prompts@^2.0.14":
   version "2.0.14"
   resolved "https://registry.yarnpkg.com/@types/prompts/-/prompts-2.0.14.tgz#10cb8899844bb0771cabe57c1becaaaca9a3b521"
@@ -516,6 +516,14 @@ cli-cursor@^3.1.0:
   dependencies:
     restore-cursor "^3.1.0"
 
+cli-progress@^3.9.1:
+  version "3.9.1"
+  resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.9.1.tgz#a22eba6a20f53289fdd05d5ee8cb2cc8c28f866e"
+  integrity sha512-AXxiCe2a0Lm0VN+9L0jzmfQSkcZm5EYspfqXKaSIQKqIk+0hnkZ3/v1E9B39mkD6vYhKih3c/RPsJBSwq9O99Q==
+  dependencies:
+    colors "^1.1.2"
+    string-width "^4.2.0"
+
 cli-spinners@^2.5.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-2.6.0.tgz#36c7dc98fb6a9a76bd6238ec3f77e2425627e939"
@@ -571,6 +579,11 @@ color@^4.0.1:
     color-convert "^2.0.1"
     color-string "^1.6.0"
 
+colors@^1.1.2:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
+  integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
+
 commander@^6.2.1:
   version "6.2.1"
   resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
@@ -1687,11 +1700,6 @@ ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
-mute-stream@~0.0.4:
-  version "0.0.8"
-  resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
-  integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
-
 napi-build-utils@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806"
@@ -1977,13 +1985,6 @@ progress@^2.0.0:
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
 
-promptly@^3.2.0:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/promptly/-/promptly-3.2.0.tgz#a5517fbbf59bd31c1751d4e1d9bef1714f42b9d8"
-  integrity sha512-WnR9obtgW+rG4oUV3hSnNGl1pHm3V1H/qD9iJBumGSmVsSC5HpZOLuu8qdMb6yCItGfT7dcRszejr/5P3i9Pug==
-  dependencies:
-    read "^1.0.4"
-
 prompts@^2.4.1:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.1.tgz#befd3b1195ba052f9fd2fde8a486c4e82ee77f61"
@@ -2051,13 +2052,6 @@ read-pkg@^3.0.0:
     normalize-package-data "^2.3.2"
     path-type "^3.0.0"
 
-read@^1.0.4:
-  version "1.0.7"
-  resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4"
-  integrity sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=
-  dependencies:
-    mute-stream "~0.0.4"
-
 readable-stream@^2.0.6:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@@ -2311,7 +2305,7 @@ string-width@^1.0.1:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^4.0.0"
 
-string-width@^4.2.3:
+string-width@^4.2.0, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==