Fix development setup to support web workers (#524)

src/ensembl/scripts/start.ts

@@ -46,13 +46,6 @@ const start = async () => {
   webpackClientConfig.output.hotUpdateChunkFilename =
     'updates/[id].[hash].hot-update.js';
 
-  webpackClientConfig.output.publicPath = [
-    `${DEVSERVER_HOST}:${WEBPACK_PORT}`,
-    webpackClientConfig.output.publicPath
-  ]
-    .join('/')
-    .replace(/([^:+])\/+/g, '$1/');
-
   console.log('ABOUT TO COMPILE'); // eslint-disable-line no-console
 
   const multiCompiler = webpack([webpackClientConfig, webpackServerConfig]);
@@ -64,9 +57,9 @@ const start = async () => {
     once(() => {
       // Not sure if this message is sufficiently visible
       setTimeout(
-        () => console.log('Starting the server; please wait...'),
+        () => console.log('Starting the server; please wait...'), // eslint-disable-line no-console
         1000
-      ); // eslint-disable-line no-console
+      );
       serverCompiler.watch(
         {},
         once(() => {
@@ -82,7 +75,13 @@ const start = async () => {
     return next();
   });
 
-  app.use(webpackDevMiddleware(clientCompiler));
+  app.use(
+    webpackDevMiddleware(clientCompiler, {
+      // according to the docs, the middleware should be able to pick the publicPath straight from the clientCompiler,
+      // but it refuses to do so, hence the manually passed option – TODO: investigate
+      publicPath: '/static/'
+    })
+  );
 
   app.use(webpackHotMiddleware(clientCompiler));
 

src/ensembl/src/server/middleware/devMiddleware.ts → src/ensembl/src/server/middleware/proxyMiddleware.ts

@@ -59,7 +59,7 @@ genomeBrowserRouter.get(
  * while directing requests for help&docs api to your locally running server,
  * use the following configuraiton:
 
-const proxyMiddleware = createProxyMiddleware(['/api/**', '!/api/docs/**'], {
+const apiProxyMiddleware = createProxyMiddleware(['/api/**', '!/api/docs/**'], {
   target: 'https://staging-2020.ensembl.org',
   changeOrigin: true,
   secure: false
@@ -74,16 +74,28 @@ const docsProxyMiddleware = createProxyMiddleware('/api/docs/**', {
   secure: false
 });
 
-const devMiddleware = [genomeBrowserRouter, proxyMiddleware, docsProxyMiddleware];
+const proxyMiddleware = [apiProxyMiddleware, docsProxyMiddleware];
 
 */
 
-const proxyMiddleware = createProxyMiddleware('/api', {
+// proxy all requests for static assets to the server that runs webpack dev middleware
+const staticAssetsMiddleware = createProxyMiddleware('/static', {
+  target: 'http://localhost:8081'
+});
+
+const apiProxyMiddleware = createProxyMiddleware('/api', {
   target: 'https://staging-2020.ensembl.org',
   changeOrigin: true,
   secure: false
 });
 
-const devMiddleware = [genomeBrowserRouter, proxyMiddleware];
+let proxyMiddleware = [apiProxyMiddleware];
+
+if (process.env.NODE_ENV === 'development') {
+  proxyMiddleware = proxyMiddleware.concat([
+    genomeBrowserRouter, // NOTE: this middleware should have priority over staticAssetsMiddleware
+    staticAssetsMiddleware
+  ]);
+}
 
-export default devMiddleware;
+export default proxyMiddleware;

src/ensembl/src/server/routes/viewsRouter.tsx

@@ -93,7 +93,6 @@ const viewRouter = async (req: Request, res: Response) => {
       <base href="/">
       ${helmet.title.toString()}
       ${helmet.meta.toString()}
-      <link rel="manifest" href="/static/manifest.json">
 
       ${extractor.getLinkTags()}
       ${extractor.getStyleTags()}

src/ensembl/src/server/server.ts

@@ -16,15 +16,21 @@
 
 import express from 'express';
 
-import devMiddleware from './middleware/devMiddleware';
+import proxyMiddleware from './middleware/proxyMiddleware';
 import staticMiddleware from './middleware/staticMiddleware';
 import viewsRouter from './routes/viewsRouter';
 
 const app = express();
 
 app.disable('x-powered-by'); // no need to announce to the world that we are running on Express
-app.use(devMiddleware);
-app.use('/static', staticMiddleware);
+
+app.use(proxyMiddleware);
+
+if (process.env.NODE_ENV === 'production') {
+  // should be able to handle requests for the contents of /static directory by itself
+  // (even though in real production deployment, requests for /static will be routed to an nginx container)
+  app.use('/static', staticMiddleware);
+}
 
 // All GET requests not covered by the middleware above will be handled by the viewsRouter
 app.get('*', viewsRouter);

src/ensembl/src/shared/components/instant-download/instant-download-fetch/fetchForGene.ts

@@ -26,10 +26,7 @@ import {
   prepareDownloadParameters
 } from './fetchForTranscript';
 
-// @ts-expect-error There is in fact no default export in the worker
-import SequenceFetcherWorker, {
-  WorkerApi
-} from 'src/shared/workers/sequenceFetcher.worker';
+import type { WorkerApi } from 'src/shared/workers/sequenceFetcher.worker';
 
 type GeneOptions = {
   transcript: Partial<TranscriptOptions>;
@@ -73,7 +70,9 @@ export const fetchForGene = async (payload: FetchPayload) => {
     );
   }
 
-  const worker = new SequenceFetcherWorker();
+  const worker = new Worker(
+    new URL('src/shared/workers/sequenceFetcher.worker.ts', import.meta.url)
+  );
 
   const service = wrap<WorkerApi>(worker);
   const sequences = await service.downloadSequences(sequenceDownloadParams);

src/ensembl/src/shared/components/instant-download/instant-download-fetch/fetchForProtein.ts

@@ -27,8 +27,7 @@ import {
   TranscriptSequenceMetadata
 } from './fetchSequenceChecksums';
 
-// @ts-expect-error There is in fact no default export in the worker
-import SequenceFetcherWorker, {
+import {
   WorkerApi,
   SingleSequenceFetchParams
 } from 'src/shared/workers/sequenceFetcher.worker';
@@ -51,7 +50,9 @@ export const fetchForProtein = async (payload: FetchPayload) => {
     options
   });
 
-  const worker = new SequenceFetcherWorker();
+  const worker = new Worker(
+    new URL('src/shared/workers/sequenceFetcher.worker.ts', import.meta.url)
+  );
 
   const service = wrap<WorkerApi>(worker);
 

src/ensembl/src/shared/components/instant-download/instant-download-fetch/fetchForTranscript.ts

@@ -29,8 +29,7 @@ import {
   TranscriptSequenceMetadata
 } from './fetchSequenceChecksums';
 
-// @ts-expect-error There is in fact no default export in the worker
-import SequenceFetcherWorker, {
+import type {
   WorkerApi,
   SingleSequenceFetchParams
 } from 'src/shared/workers/sequenceFetcher.worker';
@@ -75,7 +74,9 @@ export const fetchForTranscript = async (payload: FetchPayload) => {
     );
   }
 
-  const worker = new SequenceFetcherWorker();
+  const worker = new Worker(
+    new URL('src/shared/workers/sequenceFetcher.worker.ts', import.meta.url)
+  );
 
   const service = wrap<WorkerApi>(worker);
 

src/ensembl/static/html/template.html

-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-  <meta charset="utf-8">
-  <title>Ensembl</title>
-  <base href="/">
-  <link rel="manifest" href="/static/manifest.json">
-
-  <link rel="apple-touch-icon" sizes="180x180" href="/static/favicons/apple-touch-icon.png">
-  <link rel="icon" type="image/png" sizes="32x32" href="/static/favicons/favicon-32x32.png">
-  <link rel="icon" type="image/png" sizes="16x16" href="/static/favicons/favicon-16x16.png">
-  <link rel="mask-icon" href="/static/favicons/safari-pinned-tab.svg" color="#5bbad5">
-  <link rel="shortcut icon" href="/static/favicons/favicon.ico">
-  <meta name="msapplication-TileColor" content="#da532c">
-  <meta name="msapplication-config" content="/static/favicons/browserconfig.xml">
-  <meta name="theme-color" content="#ffffff">
-</head>
-
-<body>
-  <div id="ens-app" class="ens-app"></div>
-
-  <script src="https://polyfill.io/v3/polyfill.min.js?features=AbortController%2Object.assign%2CPromise%2Cfetch%2CIntersectionObserver%2CIntersectionObserverEntry%2CResizeObserver"></script>
-</body>
-
-</html>

src/ensembl/static/manifest.json

-{
-  "short_name": "Ensembl",
-  "name": "Ensembl website",
-  "icons": [
-    {
-      "src": "/static/favicons/android-chrome-192x192.png",
-      "sizes": "192x192",
-      "type": "image/png"
-    },
-    {
-        "src": "/static/favicons/android-chrome-512x512.png",
-        "sizes": "512x512",
-        "type": "image/png"
-    }
-  ]
-}

src/ensembl/webpack/client/common.ts

@@ -34,10 +34,6 @@ export default (env: Record<string, unknown>): Configuration => {
 
     module: {
       rules: [
-        {
-          test: /\.worker\.ts$/,
-          use: [{ loader: 'worker-loader' }, { loader: 'babel-loader' }]
-        },
         {
           test: /.tsx?$/,
           loader: 'babel-loader',
@@ -87,15 +83,12 @@ export default (env: Record<string, unknown>): Configuration => {
     output: {
       // dev will take the default file names as no physical files are emitted
       // prod will have emitted files and will include the content hash, which will change every time the contents of the js file changes.
-      filename: isDev ? undefined : '[name].[contenthash].js',
+      filename: isDev ? '[name].js' : '[name].[contenthash].js',
 
       path: paths.buildStaticPath,
 
-      // stop webpack from adding additional comments/info to generated bundles as it is a performance hit (slows down build times)
-      pathinfo: false,
-
       // prepend the public path as the root path to all the files that are inserted into the index file
-      publicPath: isDev ? '/' : '/static/'
+      publicPath: '/static/'
     },
 
     plugins: [

src/ensembl/webpack/client/development.ts

@@ -30,11 +30,7 @@ export default (): Configuration => ({
       // this is the loader that will make webpack load file formats that are not supported by other loaders
       {
         test: /\.(woff|woff2|eot|ttf|otf|gif|png|jpe?g)$/,
-        loader: 'file-loader',
-        options: {
-          // the file path and name that webpack will use to store these files
-          name: `[path][name].[ext]`
-        }
+        type: 'asset/resource'
       }
     ]
   },

src/ensembl/webpack/client/production.ts

@@ -48,31 +48,23 @@ export default (): Configuration => {
     devtool: 'source-map',
     module: {
       rules: [
-        // loader for images
-        // image loader should compress the images
-        // then file loader takes over to copy the images into the dist folder
         {
           test: /.*\.(gif|png|jpe?g)$/i,
+          type: 'asset/resource',
+          generator: {
+            filename: 'images/[name].[hash][ext]'
+          },
           use: [
-            {
-              loader: 'file-loader',
-              options: {
-                emitFile: true,
-                name: '[name].[hash].[ext]',
-                outputPath: 'images'
-              }
-            },
+            // image loader should compress the images
             'image-webpack-loader'
           ]
         },
 
-        // loader for fonts that copies the fonts into the dist folder
         {
           test: /static\/fonts\/.*\.(woff2?|eot|ttf|otf|svg)$/i,
-          loader: 'file-loader',
-          options: {
-            emitFile: true,
-            name: '[path][name].[hash].[ext]'
+          type: 'asset/resource',
+          generator: {
+            filename: 'fonts/[name].[hash][ext]'
           }
         }
       ]
@@ -103,10 +95,6 @@ export default (): Configuration => {
             from: path.join(paths.staticPath, 'favicons/*'),
             to: path.join(paths.buildStaticPath, 'favicons', '[name][ext]')
           },
-          {
-            from: path.join(paths.staticPath, 'manifest.json'),
-            to: path.join(paths.buildStaticPath, '[name][ext]')
-          },
           {
             from: path.join(paths.staticPath, 'robots.txt'),
             to: paths.buildPath