Commit e38a5b14 authored by Andrey Azov's avatar Andrey Azov Committed by Dan Sheppard
Browse files

Import genome browser static files from a github repository (#181)

* Import genome browser static files from a github repository
* Send deployed assets to assets repo.
* Delete module cache to force npm to take a look.
* Pruning script
* Confirmation for push
* QA.
* Update package-lock.json
* Switch to using tarballs.
parent 434a7969
Pipeline #45586 passed with stages
in 5 minutes and 30 seconds
......@@ -29,6 +29,10 @@ case "$1" in
echo "Running deploy check"
$BASE/../deploy.sh check
;;
deploy-strip)
echo "Running deploy strip"
$BASE/../deploy.sh strip
;;
*)
echo "Unknown command '$1'"
;;
......
......@@ -2,13 +2,23 @@
set -ev
ASSET_REPO_NAME="ensembl-genome-browser-assets"
ASSET_REPO="git@github.com:Ensembl/$ASSET_REPO_NAME.git"
RAW_ASSET_BASE="https://raw.githubusercontent.com/Ensembl/$ASSET_REPO_NAME/master"
# Where is this script running?
pushd $(dirname "${0}") > /dev/null
BASE=$(pwd -L)
popd > /dev/null
# Source and destination directories
SRC="$BASE/.."
DEST="$BASE/../../../../ensembl/static/browser"
cd $SRC
echo "SRC=$SRC"
echo "DEST=$DEST"
# What compile flags are needed?
touch Cargo.lock # force build
if [ "$1" == "check" ] ; then
export RUSTFLAGS="--cfg=deploy --cfg=console -A warnings"
......@@ -16,29 +26,86 @@ else
export RUSTFLAGS="--cfg=deploy"
fi
# Compile
cargo +nightly web build --target=wasm32-unknown-unknown --release
cargo +nightly web deploy --target=wasm32-unknown-unknown --release
echo "SRC=$SRC"
echo "DEST=$DEST"
# Compute WASMHASH
if hash md5 2>/dev/null; then
WASMHASH=$(md5 $SRC/target/deploy/hellostdweb.wasm | cut -f4 -d' ')
else
WASMHASH=$(md5sum $SRC/target/deploy/hellostdweb.wasm | cut -f1 -d' ')
fi
echo "WASMHASH=$WASMHASH"
# Update files and copy/optimise into place
WASMNAME="browser-$WASMHASH.wasm"
JSNAME="browser.js"
rm -f $DEST/*.js $DEST/*.wasm
if [ "$1" != "check" ] && hash wasm-opt 2>/dev/null ; then
wasm-opt -Os $SRC/target/deploy/hellostdweb.wasm -o $DEST/$WASMNAME
else
cp $SRC/target/deploy/hellostdweb.wasm $DEST/$WASMNAME
# Fix hash in js
sed -i -e "s~\"hellostdweb.wasm\"~\"/static/browser/$WASMNAME\"~g" $SRC/target/deploy/hellostdweb.js
# Send to remote repository
NOW=$(date +"%Y-%m-%d %H:%M")
BRANCH="$(git symbolic-ref HEAD 2>/dev/null)" || BRANCH="(anon. branch)"
BRANCH=${BRANCH##refs/heads/}
MODDIR="$SRC/../../../ensembl/node_modules/ensembl-genome-browser"
if [ "x$1" == "x" ] ; then
if hash wasm-opt 2>/dev/null ; then
echo "Found wasm-opt: good"
else
echo "Refusing to deploy unoptimised WASM"
exit 1
fi
echo "Making tarball"
TARDIR=$(mktemp -d)
(
set -ev
cd $TARDIR
mkdir package
cd package
wasm-opt -Os $SRC/target/deploy/hellostdweb.wasm -o $WASMNAME
cp $SRC/target/deploy/hellostdweb.js $JSNAME
cp $SRC/build/package.json package.json
cd ..
tar -c -z -f assets-$WASMHASH.tar.gz package
cp assets-$WASMHASH.tar.gz /tmp
)
echo "Sending to remote asset repo"
GITDIR=$(mktemp -d)
echo $GITDIR
(
set -ev
cd $GITDIR
git clone $ASSET_REPO
cd $ASSET_REPO_NAME
cp $TARDIR/assets-$WASMHASH.tar.gz .
git add assets-$WASMHASH.tar.gz
git commit -m "$BRANCH at $NOW"
git push
)
rm -rf $GITDIR $TARDIR $MODDIR
RAW_URL="$RAW_ASSET_BASE/assets-$WASMHASH.tar.gz"
(
set -ev
cd "$SRC/../../../ensembl"
sed -i -e "s~.*ensembl-genome-browser.*~ \"ensembl-genome-browser\": \"$RAW_URL\",~" package.json
npm install || true
2>&1 echo
(git diff package-lock.json | grep '^[+-]' | grep -v ensembl-genome-browser-assets | grep -v -- --- | grep -v +++ | grep -v '"integrity":') && ( 2>&1 echo "ERROR! package lock did not update cleanly, aborting" && exit 1 )
) || exit 1
elif [ "x$1" == "xstrip" ] ; then
rm -f $MODDIR/browser.js $MODDIR/browser-*.wasm
if hash wasm-opt 2>/dev/null ; then
wasm-opt -Os $SRC/target/deploy/hellostdweb.wasm -o $MODDIR/$WASMNAME
else
cp $SRC/target/deploy/hellostdweb.wasm $MODDIR/$WASMNAME
fi
cp $SRC/target/deploy/hellostdweb.js $MODDIR/$JSNAME
elif [ "x$1" == "xcheck" ] ; then
rm -f $MODDIR/browser.js $MODDIR/browser-*.wasm
cp $SRC/target/deploy/hellostdweb.wasm $MODDIR/$WASMNAME
cp $SRC/target/deploy/hellostdweb.js $MODDIR/$JSNAME
fi
ls -lh $DEST/$WASMNAME
cp $SRC/target/deploy/hellostdweb.js $DEST/$JSNAME
sed -i -e "s~\"hellostdweb.wasm\"~\"/static/browser/$WASMNAME\"~g" $DEST/$JSNAME
{
"name": "ensembl-genome-browser",
"version": "1.0.0",
"description": "Genome browser for Ensembl web site built as static assets.",
"main": "browser.js",
"license": "MIT"
}
#! /usr/bin/env python3
# This script can be used to prune the ensembl-static-assets repo, removing
# any WASM files which are unreferenced from ensembl-client. This is so
# that this repo can be GCed and so kept to a sensible size. Note that this
# script doesn't actually do the GC-ing itself, leaving it to you or to
# github.
#
# The script works out what needs to be kept by inspecting all versions of
# package.json anywhere in the ensmebl-client repo. For each of these it
# extracts the tag in use in that package.json. At this stage it also inspects
# "VIP" branches (master and dev for now) and inspects which WASMs it
# DEFINITELY doesn't want to delete as a double check for later. (They should
# never be deleted by this script anyway).
#
# The main tidying begins with a new commit which removes unused tar files.
# When that's committed, a "non-interactive" interactive rebase squashes
# everything down to a single commit (to remove all references to the deleted
# files, allowing them to be collected).
#
# When this is done, the log is presented to the user for inspection and if
# the user agrees the updated repo is force-pushed. There's then a final
# fiddly step to remove tags on the remote (necessary for github to gc our
# repo) by checking out the newly-pushed repo again and comparing the tags
# which are there to what we expect and removing all others.
import collections, optparse, os, re, subprocess, sys, tempfile
CLIENT_REPO_NAME="ensembl-client"
CLIENT_REPO="https://github.com/Ensembl/{0}.git"
ASSET_REPO_NAME="ensembl-genome-browser-assets"
ASSET_REPO="git@github.com:Ensembl/{0}.git"
parser = optparse.OptionParser()
parser.add_option("-n", "--dry-run", action="store_true", dest="dry", help="dry run", default=False)
(options, args) = parser.parse_args()
vip_branches = set(["master","dev"])
self_dir = os.path.dirname(os.path.abspath(__file__))
# Generic means of running stuff
def run(*args,**kwargs):
if 'stderr' not in kwargs:
kwargs['stderr'] = subprocess.PIPE
fail_ok = False
if 'fail_ok' in kwargs:
fail_ok = kwargs['fail_ok']
del kwargs['fail_ok']
print(" ".join(args[0]))
proc = subprocess.run(*args,**kwargs)
if proc.stderr != None:
sys.stderr.write(proc.stderr.decode("utf8"))
if proc.returncode != 0 and not fail_ok:
sys.stderr.write("Cannot run {0} {1}".format(args,kwargs))
sys.exit(1)
return proc.stdout.decode("utf8") if proc.stdout else ""
# Extract wasm-in-use tag from package.json blob
depline_re = re.compile(r'.*"ensembl-genome-browser":.*/assets-(.*?)\..*"')
def extract_tag(blob):
contents = run(["git","cat-file","-p",blob],stdout=subprocess.PIPE)
for line in contents.split("\n"):
m = depline_re.match(line)
if m:
return m.group(1)
return None
# Extract all wasms referred to in client repo
def list_tags():
out = []
blobs = run(["git","rev-list","--all","--objects","src/ensembl/package.json"],stdout=subprocess.PIPE)
for blob in blobs.split("\n"):
parts = blob.strip().split(' ',1)
if len(parts) < 2 or parts[1] != "src/ensembl/package.json":
continue
tag = extract_tag(parts[0])
if tag:
out.append(tag)
return set(out)
# List all assets in asset repo
asset_tar_re = re.compile(r'assets-(.*)\.tar\.gz')
def list_assets():
out = []
for filename in os.listdir("."):
m = asset_tar_re.match(filename)
if m and m.group(1):
out.append(m.group(1))
return set(out)
# ensembl-client repo investigation
tags_in_use = set()
vip_wasms = {}
with tempfile.TemporaryDirectory() as tmpdirname:
os.chdir(tmpdirname)
repo = CLIENT_REPO.format(CLIENT_REPO_NAME)
print("tmpdir is {0}".format(tmpdirname))
run(["git","clone",repo])
os.chdir(CLIENT_REPO_NAME)
# Get list of tags in use
tags_in_use = list_tags()
# get VIP wasms (used for warnings)
for branch in vip_branches:
run(["git","checkout",branch])
blobs = run(["git","ls-tree",branch,"--full-tree","src/ensembl/package.json"],stdout=subprocess.PIPE,fail_ok=True)
for blob in blobs.split("\n"):
parts = blob.split()
if len(parts) < 4:
continue
if parts[1] == 'blob' and parts[3] == 'src/ensembl/package.json':
vip_wasms[branch] = extract_tag(parts[2])
lines =(["","tags in use"]+sorted(list(tags_in_use))+
["","VIP tags"]+[ "{0} {1}".format(*x) for x in vip_wasms.items() ]+
[""])
print("\n".join(lines))
# y/n for user
def confirm(msg):
while True:
confirm = input(msg).lower().strip()
if confirm.startswith("n"):
sys.stderr.write("Abandoning at your request\n")
sys.exit(1)
elif confirm.startswith("y"):
break
else:
sys.stderr.write("Eh?\n")
with tempfile.TemporaryDirectory() as tmpdirname:
# Checkout asset repo
os.chdir(tmpdirname)
repo = ASSET_REPO.format(ASSET_REPO_NAME)
print("tmpdir is {0}".format(tmpdirname))
run(["git","clone",repo])
os.chdir(ASSET_REPO_NAME)
# Find, delete, commit unused assets
all_wasms = list_assets()
for old_wasm in all_wasms - tags_in_use:
filename = "assets-{0}.tar.gz".format(old_wasm)
run(["rm",filename])
run(["git","rm",filename])
print("{0} assets in use; {1} no longer in use".format(
len(tags_in_use),len(all_wasms-tags_in_use)
))
run(["git","commit","-m","Repo prune"])
# Find first commit
origin = run(["git","log","--max-parents=0","--pretty=format:%H"],stdout=subprocess.PIPE).split("\n")[0]
print("first commit is {0}".format(origin))
# Squash everything to single commit
prune_squasher = os.path.join(self_dir,"prune-squasher.py")
env = os.environ.copy()
env['GIT_SEQUENCE_EDITOR'] = prune_squasher
env['GIT_EDITOR'] = "cat"
run(["git","rebase","-i",origin],env=env)
run(["git","commit","--amend","-m","wasm assets"])
run(['git','log'])
run(["ls","-l"])
# Check VIP branch wasms are still in the repo
tags_in_repo = list_assets()
warnings = False
sys.stderr.write("\n")
for (branch,wasm) in vip_wasms.items():
if wasm in tags_in_repo:
sys.stderr.write("{0}'s wasm ({1}) is in repo. Good!\n".format(branch,wasm[:13]))
else:
sys.stderr.write("WARNING! {0}'s wasm ({1}) would not be in rebased repo!\n".format(branch,wasm))
warnings = True
if warnings:
sys.stderr.write("\n\nwasms missing from repo. Continuing at this point may break 2020 live, deployed site. Strongly suggest not coninuing\n\n")
confirm("continue and maybe break live site? ")
# log and confirm
run(["git","log"])
if options.dry:
sys.stderr.write("Exiting as was dry run\n")
sys.exit(1)
confirm("Is this ok to push? ")
# push
run(["git","push","--force"])
#! /usr/bin/env python3
import re, sys
if len(sys.argv) != 2:
sys.stderr.write("Must supply exactly one argument. got {0}".format(sys.argv))
sys.exit(1)
def match(subhash):
for squash in squashes:
if squash.startswith(subhash):
return True
return False
new_contents = []
pick_re = re.compile(r'pick\s+(\S+)\s')
with open(sys.argv[1]) as edit_file:
for (i,line) in enumerate(edit_file.readlines()):
m = pick_re.match(line)
if m and i>0:
line = pick_re.sub("squash {0} ".format(m.group(1)),line)
new_contents.append(line)
with open(sys.argv[1],"w") as edit_file:
edit_file.write("".join(new_contents))
sys.stderr.write("".join(new_contents))
......@@ -7618,6 +7618,10 @@
}
}
},
"ensembl-genome-browser": {
"version": "https://raw.githubusercontent.com/Ensembl/ensembl-genome-browser-assets/master/assets-cb6bfc9169ea4106c887a2f5c2b234ac.tar.gz",
"integrity": "sha512-nHFES3rGJd2ENmyF6laX0Sljf348CZDjDYYIiXZlX088BYNLOgL1rhJGiG2dxVTpLwsgrGUxcEyuv907zVJazg=="
},
"entities": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
......
......@@ -54,6 +54,7 @@
"connected-react-router": "6.5.2",
"core-js": "3.3.6",
"dotenv": "8.2.0",
"ensembl-genome-browser": "https://raw.githubusercontent.com/Ensembl/ensembl-genome-browser-assets/master/assets-cb6bfc9169ea4106c887a2f5c2b234ac.tar.gz",
"koa-proxy": "1.0.0-alpha.3",
"lodash": "4.17.15",
"query-string": "6.8.3",
......
......@@ -16,7 +16,7 @@ jest.mock('./track-panel/TrackPanel', () => () => <div>TrackPanel</div>);
jest.mock('./browser-app-bar/BrowserAppBar', () => () => (
<div>BrowserAppBar</div>
));
jest.mock('static/browser/browser.js', () => {});
jest.mock('ensembl-genome-browser', () => {});
describe('<Browser />', () => {
afterEach(() => {
......
......@@ -56,7 +56,7 @@ import * as urlFor from 'src/shared/helpers/urlHelper';
import styles from './Browser.scss';
import 'static/browser/browser.js';
import 'ensembl-genome-browser';
export type BrowserProps = {
activeGenomeId: string | null;
......
This diff is collapsed.
......@@ -2,6 +2,7 @@ const dotenv = require('dotenv').config();
const webpack = require('webpack');
const path = require('path');
const url = require('url');
const StylelintWebpackPlugin = require('stylelint-webpack-plugin');
// laoders specific to dev
......@@ -49,14 +50,16 @@ const devConfig = {
before(app) {
// use proper mime-type for wasm files
app.get('*.wasm', function(req, res, next) {
var options = {
root: path.join(__dirname, '..'),
const options = {
root: path.join(__dirname, '../node_modules/ensembl-genome-browser'),
dotfiles: 'deny',
headers: {
'Content-Type': 'application/wasm'
}
};
res.sendFile(req.url, options, function(err) {
const parsedUrl = url.parse(req.url);
const fileName = path.basename(parsedUrl.pathname);
res.sendFile(fileName, options, function(err) {
if (err) {
next(err);
}
......
......@@ -66,7 +66,7 @@ const plugins = [
// copy static assets
new CopyWebpackPlugin([
{
from: path.join(__dirname, '../static/browser/browser*.wasm'),
from: path.join(__dirname, '../node_modules/ensembl-genome-browser/browser*.wasm'),
to: path.join(__dirname, '../dist/static/browser/'),
flatten: true
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment