ZenBrowserMirrors/surfer
1
0
Fork 0
mirror of https://github.com/zen-browser/surfer.git synced 2025-07-08 09:20:01 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

🎨 Refactor the download system some

Browse source
This commit is contained in:
trickypr 2021-11-12 18:44:20 +11:00
parent f23019e266
commit 775cf7ec75
7 changed files with 154 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -38,6 +38,7 @@
"dependencies": {
"axios": "^0.21.1",
"chalk": "^4.1.0",
"cli-progress": "^3.9.1",
"commander": "^6.2.1",
"execa": "^5.1.1",
"fs-extra": "^9.0.1",
@ -49,6 +50,7 @@
"sharp": "^0.29.1"
},
"devDependencies": {
"@types/cli-progress": "^3.9.2",
"@types/fs-extra": "^9.0.6",
"@types/node": "^14.14.16",
"@types/prompts": "^2.0.14",

4
src/cmds.ts
View file

@ -103,8 +103,8 @@ export const commands: Cmd[] = [
controller: importPatches,
},
{
cmd: 'init <source>',
aliases: ['initialise', 'initialize'],
cmd: 'ff-init <source>',
aliases: ['ff-initialise', 'ff-initialize'],
description: 'Initialise the Firefox directory.',
controller: init,
},

33
src/commands/download-artifacts.ts
View file

@ -1,11 +1,10 @@
import axios from 'axios'
import execa from 'execa'
import fs from 'fs'
import { homedir } from 'os'
import { posix, resolve, sep } from 'path'
import { log } from '..'
import { downloadFileToLocation } from '../utils/download'
export const downloadArtifacts = async () => {
export const downloadArtifacts = async (): Promise<void> => {
if (process.platform !== 'win32')
return log.error(
'This is not a Windows machine, will not download artifacts.'
@ -25,37 +24,11 @@ export const downloadArtifacts = async () => {
log.info(`Downloading Windows artifacts...`)
const { data, headers } = await axios.get(url, {
responseType: 'stream',
})
await downloadFileToLocation(url, resolve(process.cwd(), filename))
const length = headers['content-length']
const writer = fs.createWriteStream(resolve(process.cwd(), filename))
let receivedBytes = 0
data.on('data', (chunk: any) => {
receivedBytes += chunk.length
const rand = Math.floor(Math.random() * 1000 + 1)
if (rand > 999.5) {
const percentCompleted = parseInt(
Math.round((receivedBytes * 100) / length).toFixed(0)
)
if (percentCompleted % 2 == 0 || percentCompleted >= 100) return
log.info(`\t${filename}\t${percentCompleted}%...`)
}
})
data.pipe(writer)
data.on('end', async () => {
log.info('Unpacking mozbuild...')
await execa('tar', ['-xvf', filename, '-C', home])
log.info('Done extracting mozbuild artifacts.')
})
}

198
src/commands/download.ts
View file

@ -1,14 +1,14 @@
import axios from 'axios'
import chalk from 'chalk'
import execa from 'execa'
import fs, { existsSync, rmdirSync, writeFileSync } from 'fs'
import fs, { existsSync, rmdirSync } from 'fs'
import { ensureDirSync, removeSync } from 'fs-extra'
import ora from 'ora'
import { homedir } from 'os'
import { posix, resolve, sep } from 'path'
import { bin_name, config, log } from '..'
import { bin_name, log } from '..'
import { ENGINE_DIR } from '../constants'
import { getConfig, getLatestFF, writeMetadata } from '../utils'
import { downloadFileToLocation } from '../utils/download'
import { downloadArtifacts } from './download-artifacts'
const gFFVersion = getConfig().version.version
@ -23,6 +23,78 @@ let initProgress: any = ora({
indent: 0,
})
export const download = async () => {
const version = gFFVersion
// If gFFVersion isn't specified, provide legible error
if (!version) {
log.error(
'You have not specified a version of firefox in your config file. This is required to build a firefox fork'
)
process.exit(1)
}
// The location to download the firefox source code from the web
const base = `https://archive.mozilla.org/pub/firefox/releases/${version}/source/`
const filename = `firefox-${version}.source.tar.xz`
const url = base + filename
log.info(`Locating Firefox release ${version}...`)
ensureDirSync(resolve(process.cwd(), `.dotbuild`, `engines`))
if (
existsSync(
resolve(
process.cwd(),
`.dotbuild`,
`engines`,
`firefox-${version.split('b')[0]}`
)
)
) {
log.error(
`Cannot download version ${
version.split('b')[0]
} as it already exists at "${resolve(
process.cwd(),
`firefox-${version.split('b')[0]}`
)}"`
)
}
if (version.includes('b'))
log.warning(
'Version includes non-numeric characters. This is probably a beta.'
)
// Do not re-download if there is already an existing workspace present
if (existsSync(ENGINE_DIR)) {
log.error(
`Workspace already exists.\nRemove that workspace and run |${bin_name} download ${version}| again.`
)
}
log.info(`Downloading Firefox release ${version}...`)
await downloadFileToLocation(
url,
resolve(process.cwd(), `.dotbuild`, `engines`, filename)
)
await unpack(filename, version)
if (process.platform === 'win32') {
if (existsSync(resolve(homedir(), '.mozbuild'))) {
log.info('Mozbuild directory already exists, not redownloading')
} else {
log.info('Mozbuild not found, downloading artifacts.')
await downloadArtifacts()
}
}
}
const onData = (data: any) => {
const d = data.toString()
@ -72,7 +144,7 @@ const unpack = async (name: string, version: string) => {
tarProc.on('exit', () => {
if (process.env.CI_SKIP_INIT) return log.info('Skipping initialisation.')
const initProc = execa('npx', ['melon', 'init', 'engine'])
const initProc = execa('npx', ['melon', 'ff-init', 'engine'])
;(initProc.stdout as any).on('data', onData)
;(initProc.stdout as any).on('error', onData)
@ -82,8 +154,6 @@ const unpack = async (name: string, version: string) => {
initProgress.stop()
initProgress = null
await new Promise((resolve) => setTimeout(resolve, 5000))
log.success(
`You should be ready to make changes to Dot Browser.\n\n\t You should import the patches next, run |${bin_name} import|.\n\t To begin building Dot, run |${bin_name} build|.`
)
@ -97,119 +167,3 @@ const unpack = async (name: string, version: string) => {
})
})
}
export const download = async (firefoxVersion?: string) => {
if (firefoxVersion)
log.warning(
`A custom Firefox version is being used. Some features of Dot may not work as expected.`
)
if (!firefoxVersion) {
firefoxVersion = gFFVersion
}
let version = await getLatestFF()
if (firefoxVersion) {
version = firefoxVersion
}
const base = `https://archive.mozilla.org/pub/firefox/releases/${version}/source/`
const filename = `firefox-${version}.source.tar.xz`
const url = `${base}${filename}`
log.info(`Locating Firefox release ${version}...`)
ensureDirSync(resolve(process.cwd(), `.dotbuild`, `engines`))
if (
existsSync(
resolve(
process.cwd(),
`.dotbuild`,
`engines`,
`firefox-${version.split('b')[0]}`
)
)
) {
log.error(
`Cannot download version ${
version.split('b')[0]
} as it already exists at "${resolve(
process.cwd(),
`firefox-${version.split('b')[0]}`
)}"`
)
}
if (version == firefoxVersion)
log.info(`Version is frozen at ${firefoxVersion}!`)
if (version.includes('b'))
log.warning(
'Version includes non-numeric characters. This is probably a beta.'
)
if (
fs.existsSync(
resolve(
process.cwd(),
`.dotbuild`,
`engines`,
'firefox',
version.split('b')[0]
)
) ||
fs.existsSync(
resolve(process.cwd(), 'firefox', `firefox-${ version.split('b')[0]}`)
)
)
log.error(
`Workspace with version "${
version.split('b')[0]
}" already exists.\nRemove that workspace and run |${bin_name} download ${version}| again.`
)
log.info(`Downloading Firefox release ${version}...`)
const { data, headers } = await axios.get(url, {
responseType: 'stream',
})
const length = headers['content-length']
const writer = fs.createWriteStream(
resolve(process.cwd(), `.dotbuild`, `engines`, filename)
)
let receivedBytes = 0
data.on('data', (chunk: any) => {
receivedBytes += chunk.length
const rand = Math.floor(Math.random() * 1000 + 1)
if (rand > 999.5) {
const percentCompleted = parseInt(
Math.round((receivedBytes * 100) / length).toFixed(0)
)
if (percentCompleted % 2 == 0 || percentCompleted >= 100) return
log.info(`\t${filename}\t${percentCompleted}%...`)
}
})
data.pipe(writer)
data.on('end', async () => {
await unpack(filename, version)
if (process.platform === 'win32') {
if (existsSync(resolve(homedir(), '.mozbuild'))) {
log.info('Mozbuild directory already exists, not redownloading')
} else {
log.info('Mozbuild not found, downloading artifacts.')
await downloadArtifacts()
}
}
})
}

5
src/commands/init.ts
View file

@ -4,7 +4,7 @@ import { resolve } from 'path'
import { bin_name, log } from '..'
import { dispatch } from '../utils'
export const init = async (directory: Command) => {
export const init = async (directory: Command): Promise<void> => {
if (process.platform == 'win32') {
// Because Windows cannot handle paths correctly, we're just calling a script as the workaround.
log.info(
@ -41,9 +41,10 @@ export const init = async (directory: Command) => {
version = version.trim().replace(/\\n/g, '')
log.info('Initializing git, this may take some time')
await dispatch('git', ['init'], dir as string)
await dispatch('git', ['checkout', '--orphan', version], dir as string)
await dispatch('git', ['add', '-v', '-f', '.'], dir as string)
await dispatch('git', ['add', '-f', '.'], dir as string)
await dispatch(
'git',
['commit', '-am', `"Firefox ${version}"`],

43
src/utils/download.ts Normal file
View file

@ -0,0 +1,43 @@
import { createWriteStream } from 'fs'
import axios from 'axios'
import cliProgress from 'cli-progress'
export async function downloadFileToLocation(
url: string,
writeOut: string
): Promise<void> {
return new Promise((resolve, reject) =>
(async () => {
const { data, headers } = await axios.get(url, {
responseType: 'stream',
})
const length = headers['content-length']
const writer = createWriteStream(writeOut)
let receivedBytes = 0
const progressBar = new cliProgress.SingleBar({})
progressBar.start(length, receivedBytes)
data.on('data', (chunk: { length: number }) => {
receivedBytes += chunk.length
})
data.pipe(writer)
data.on('error', (err: unknown) => reject(err))
const progressInterval = setInterval(
() => progressBar.update(receivedBytes),
500
)
data.on('end', async () => {
clearInterval(progressInterval)
progressBar.stop()
resolve()
})
})()
)
}

48
yarn.lock
View file

@ -120,6 +120,13 @@
resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e"
integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA==
"@types/cli-progress@^3.9.2":
version "3.9.2"
resolved "https://registry.yarnpkg.com/@types/cli-progress/-/cli-progress-3.9.2.tgz#6ca355f96268af39bee9f9307f0ac96145639c26"
integrity sha512-VO5/X5Ij+oVgEVjg5u0IXVe3JQSKJX+Ev8C5x+0hPy0AuWyW+bF8tbajR7cPFnDGhs7pidztcac+ccrDtk5teA==
dependencies:
"@types/node" "*"
"@types/fs-extra@^9.0.6":
version "9.0.12"
resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-9.0.12.tgz#9b8f27973df8a7a3920e8461517ebf8a7d4fdfaf"
@ -160,13 +167,6 @@
resolved "https://registry.yarnpkg.com/@types/node/-/node-14.17.17.tgz#4ec7b71bbcb01a4e55455b60b18b1b6a783fe31d"
integrity sha512-niAjcewgEYvSPCZm3OaM9y6YQrL2SEPH9PymtE6fuZAvFiP6ereCcvApGl2jKTq7copTIguX3PBvfP08LN4LvQ==
"@types/promptly@^3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@types/promptly/-/promptly-3.0.2.tgz#598674d4b78b3dffcb2d756b344f28a2cf7459f8"
integrity sha512-cJFwE7d8GlraY+DJoZ0NhpoJ55slkcbNsGIKMY0H+5h0xaGqXBqXz9zeu+Ey9KfN1UiHQXiIT0GroxyPYMPP/w==
dependencies:
"@types/node" "*"
"@types/prompts@^2.0.14":
version "2.0.14"
resolved "https://registry.yarnpkg.com/@types/prompts/-/prompts-2.0.14.tgz#10cb8899844bb0771cabe57c1becaaaca9a3b521"
@ -516,6 +516,14 @@ cli-cursor@^3.1.0:
dependencies:
restore-cursor "^3.1.0"
cli-progress@^3.9.1:
version "3.9.1"
resolved "https://registry.yarnpkg.com/cli-progress/-/cli-progress-3.9.1.tgz#a22eba6a20f53289fdd05d5ee8cb2cc8c28f866e"
integrity sha512-AXxiCe2a0Lm0VN+9L0jzmfQSkcZm5EYspfqXKaSIQKqIk+0hnkZ3/v1E9B39mkD6vYhKih3c/RPsJBSwq9O99Q==
dependencies:
colors "^1.1.2"
string-width "^4.2.0"
cli-spinners@^2.5.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-2.6.0.tgz#36c7dc98fb6a9a76bd6238ec3f77e2425627e939"
@ -571,6 +579,11 @@ color@^4.0.1:
color-convert "^2.0.1"
color-string "^1.6.0"
colors@^1.1.2:
version "1.4.0"
resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
commander@^6.2.1:
version "6.2.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
@ -1687,11 +1700,6 @@ ms@^2.1.1:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
mute-stream@~0.0.4:
version "0.0.8"
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==
napi-build-utils@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806"
@ -1977,13 +1985,6 @@ progress@^2.0.0:
resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
promptly@^3.2.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/promptly/-/promptly-3.2.0.tgz#a5517fbbf59bd31c1751d4e1d9bef1714f42b9d8"
integrity sha512-WnR9obtgW+rG4oUV3hSnNGl1pHm3V1H/qD9iJBumGSmVsSC5HpZOLuu8qdMb6yCItGfT7dcRszejr/5P3i9Pug==
dependencies:
read "^1.0.4"
prompts@^2.4.1:
version "2.4.1"
resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.1.tgz#befd3b1195ba052f9fd2fde8a486c4e82ee77f61"
@ -2051,13 +2052,6 @@ read-pkg@^3.0.0:
normalize-package-data "^2.3.2"
path-type "^3.0.0"
read@^1.0.4:
version "1.0.7"
resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4"
integrity sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ=
dependencies:
mute-stream "~0.0.4"
readable-stream@^2.0.6:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@ -2311,7 +2305,7 @@ string-width@^1.0.1:
is-fullwidth-code-point "^2.0.0"
strip-ansi "^4.0.0"
string-width@^4.2.3:
string-width@^4.2.0, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==