Implemented

* HTTP & HTTPS caching * Request queue for concurrent requests
6 years ago · 35afa59721
5 changed files with 421 additions and 0 deletions
--- a/.prettierrc.js
+++ b/.prettierrc.js
@ -0,0 +1,13 @@
+/*
+ * ഓം ബ്രഹ്മാർപ്പണം 
+ * .prettierrc.js
+ * Created: Sat Mar 21 2020 22:14:26 GMT+0530 (GMT+05:30)
+ * Copyright 2020 Harish Karumuthil<harish2704@gmail.com>
+ */
+module.exports = {
+  trailingComma: "es5",
+  tabWidth: 2,
+  semi: true,
+  singleQuote: true
+};
+
--- a/README.md
+++ b/README.md
@ -0,0 +1,101 @@
+# Proxy-kutti
+Simple and transparent caching forward proxy server written in Nodejs.
+
+## Features
+* It has only one dependency ( `node-forge` )
+    - it is used for creating self-signed certificates dynamically for MITM https proxy server.
+* By default configuration, it will indefinitely cache all http requests irrespective of their cache headers.
+* All cached data is transparently saved to a cache directory with simple file structure.
+    - Contents are saved as it & headers are saved as a json file. It can be viewed/edited later
+* If Root CA certificates are provided, then MITM HTTPS proxy server will get enabled and HTTPS Traffic also will get cached.
+* We can specify URL rewrite rules to avoid caching of same data from different mirror sites.
+    - This feature will help to work with default configuration of YUM/DNF utilities which will be using different mirrors in each time.
+
+
+## Installation
+
+`npm i proxy-kutti`
+
+## Configuration
+
+* proxy-kutti will listen on `127.0.0.1:8080` by default ( without any configuration )
+* All the configuration variables can be permanently stored in configuration file
+* We can even specify different configuration file using `PROXY_KUTTI_CONFIG` environment variable.
+* All the values specified in configuration file can be overridden by setting corresponding `PROXY_KUTTI_<config key>` environment variable.
+* Multiple `url_rewrites` rules can be specified by providing space separated list of rewrite rules.
+* `url_rewrite` rule, follows format used by `s` subcommand of `sed` command.
+    The first and last charecter of the rewrite rule should be same and the same charecter is used as separator.
+* To install root CA in a CentOS-7 system, do the following
+    - `root@host# cp <rootCA.*> /etc/pki/ca-trust/source/anchors/`
+    - `root@host# update-ca-trust`
+
+
+## Example Usage
+
+`env PROXY_KUTTI_host=0.0.0.0 npx proxy-kutti`
+
+**shell Output**
+```
+Proxy-kutti is running...
+
+Using env variables
+  PROXY_KUTTI_CONFIG=/home/user/.config/proxy-kutti/config.js
+
+Current Configuration ( edit /home/user/.config/proxy-kutti/config.(json|js)  or set env variable PROXY_KUTTI_<config-key>=<value> to change )
+  "port": 8080,
+  "host": "0.0.0.0",
+  "cache_dir": "/home/user/.cache/proxy-kutti",
+  "root_ca_key": "/home/user/.config/proxy-kutti/rootCA.key",
+  "root_ca_cert": "/home/user/.config/proxy-kutti/rootCA.pem",
+  "url_rewrites": "#http://(.*)/7.7.1908/#http://mirrors.centos/7.7.1908/# #http[s]://(.*)epel/7/x86_64/#http://mirror.epel/7/x86_64/#"
+
+
+Run the following command shell to start using this proxy
+  export http_proxy=http://0.0.0.0:8080
+  export https_proxy=http://0.0.0.0:8080
+
+  
+2020-03-22T20:21:33.624Z Miss GET https://example.com/making-request-for-first-time => /home/user/.cache/proxy-kutti/https/example.com/GET/making-request-for-first-time.data
+2020-03-22T20:21:55.839Z Miss GET https://github.com/ => /home/user/.cache/proxy-kutti/https/github.com/GET/#index.data
+2020-03-22T20:26:31.251Z Miss GET https://github.com/harish2704/node-proxy-kutti/archive/master.zip => /home/user/.cache/proxy-kutti/https/github.com/GET/harish2704/node-proxy-kutti/archive/master.zip.data
+2020-03-22T20:27:23.087Z Miss GET https://example.com/again-making-those-requests => /home/user/.cache/proxy-kutti/https/example.com/GET/again-making-those-requests.data
+2020-03-22T20:27:35.043Z Hit! GET https://github.com/ => /home/user/.cache/proxy-kutti/https/github.com/GET/#index.data
+2020-03-22T20:27:38.237Z Hit! GET https://github.com/harish2704/node-proxy-kutti/archive/master.zip => /home/user/.cache/proxy-kutti/https/github.com/GET/harish2704/node-proxy-kutti/archive/master.zip.data
+
+```
+
+## Configuring MITM proxy for HTTPS traffic.
+
+To cache HTTPS traffic , a root CA certificate has to provided to proxy server.
+Then the same root CA has to be installed as a trusted CA on all the client systems.
+Otherwise "invalid issuer" error will raise during any https request.
+
+openssl command line can be used to generated Root CA certificate.
+For details please use / refer the gist [ generate-certificate-openssl.sh ](https://gist.github.com/harish2704/6cc7185c2fe36ec9cb4e912c4e74f781)
+
+Root CA certificates has to be placed in the location pointed by `root_ca_cert` & `root_ca_key` configuration values.
+
+
+## Example structure of cache directory
+
+```
+.
+└── https
+    ├── example.com
+    │   └── GET
+    │       ├── again-making-those-requests.data
+    │       ├── again-making-those-requests.data.meta
+    │       ├── making-request-for-first-time.data
+    │       └── making-request-for-first-time.data.meta
+    └── github.com
+        └── GET
+            ├── harish2704
+            │   └── node-proxy-kutti
+            │       └── archive
+            │           ├── master.zip.data
+            │           └── master.zip.data.meta
+            ├── #index.data
+            └── #index.data.meta
+```
+
+
--- a/package.json
+++ b/package.json
@ -0,0 +1,22 @@
+{
+  "name": "proxy-kutti",
+  "version": "1.0.0",
+  "description": "A Simple and transparent caching forward proxy with minimal dependencies. Cached files are stored in filesystem by preserving the structure. Cache never expires. Useful for caching various package manager files like rpm, deb etc with in local network, especially while building containers",
+  "main": "proxy.js",
+  "repository": "https://github.com/harish2704/node-proxy-kutti",
+  "author": "Harish Karumuthil <harish2704@gmail.com>",
+  "license": "MIT",
+  "bin": {
+    "proxy-kutti": "./proxy.js"
+  },
+  "dependencies": {
+    "node-forge": "^0.9.1"
+  },
+  "keywords": [
+    "proxy",
+    "caching-proxy",
+    "http-proxy",
+    "https-proxy",
+    "mitm"
+  ]
+}
--- a/proxy.js
+++ b/proxy.js
@ -0,0 +1,277 @@
+#!/usr/bin/env node
+/*
+ * ഓം ബ്രഹ്മാർപ്പണം
+ * proxy.js
+ * Created: Sat Mar 21 2020 02:04:37 GMT+0530 (GMT+05:30)
+ * Copyright 2020 Harish Karumuthil<harish2704@gmail.com>
+ */
+
+const http = require('http');
+const https = require('https');
+const url = require('url');
+const fs = require('fs');
+const util = require('util');
+const fsPromise = fs.promises;
+const { dirname } = require('path');
+const forge = require('node-forge');
+const pki = forge.pki;
+const tls = require('tls');
+const generateKeyPair = util.promisify(require('crypto').generateKeyPair);
+const net = require('net');
+const os = require('os');
+
+const httpsPort = os.tmpdir() + `/proxy-kutti-${Date.now()}.sock`
+const configHome = os.homedir() + '/.config/proxy-kutti';
+const configFile = process.env.PROXY_KUTTI_CONFIG || configHome + '/config';
+const config = {
+  port: 8080,
+  host: '127.0.0.1',
+  cache_dir: os.homedir() + '/.cache/proxy-kutti',
+  root_ca_key: configHome + '/rootCA.key',
+  root_ca_cert: configHome + '/rootCA.pem',
+  url_rewrites: '#http[s]://(.*)/7.7.1908/#http://mirrors.centos/7.7.1908/# #http[s]://(.*)epel/7/x86_64/#http://mirror.epel/7/x86_64/#',
+};
+
+try {
+  Object.assign(config, require(configFile));
+} catch (e) {}
+Object.keys(config).forEach(function(k) {
+  config[k] = process.env['PROXY_KUTTI_' + k] || config[k];
+});
+
+const urlMappings = config.url_rewrites.split(' ').map(function(pattern) {
+  debugger;
+  if (pattern[0] === pattern.slice(-1)) {
+    const [search, replace] = pattern.slice(1, -1).split(pattern[0]);
+    return { search: new RegExp(search), replace };
+  }
+  throw new Error(`Invalid url_rewrite "${pattern}"`);
+});
+
+
+function mapUrl(origUrl) {
+  let out = origUrl;
+
+  let i = 0,
+    l = urlMappings.length,
+    mapping;
+  while (i < l) {
+    urlMap = urlMappings[i];
+    out = out.replace(urlMap.search, urlMap.replace);
+    i++;
+  }
+  return out;
+}
+
+const runnninRequests = {};
+function untillRequestFinished( cachedFile ){
+  return new Promise(res => runnninRequests[ cachedFile ].on('close', res ));
+}
+function startNewRequest( cachedFile ){
+  const stream = fs.createWriteStream( cachedFile );
+  runnninRequests[cachedFile] = stream;
+  stream.on('close', () => delete runnninRequests[cachedFile] );
+  return stream;
+}
+
+async function getContent(httpModule, origReq, origRes) {
+  const origUrl = url.parse(origReq.url);
+  const mappedUrlStr = mapUrl(origReq.url);
+  const mappedUrl = url.parse(mappedUrlStr);
+  const mappedPort = mappedUrl.port ? ':' + mappedUrl.port : '';
+  const method = origReq.method;
+  const proto = httpModule === http ? 'http':'https';
+  let cachedFile = `${config.cache_dir}/${proto}/${mappedUrl.host}${mappedPort}/${method}${mappedUrl.path}`;
+  if( mappedUrl.path.slice(-1) === '/' ){
+    cachedFile += '#index.data';
+  } else {
+    cachedFile += '.data';
+  }
+  const cachedFileMeta = `${cachedFile}.meta`;
+  let proxyRes;
+  let isHit = false;
+
+
+  if( cachedFile in runnninRequests ){
+    await untillRequestFinished( cachedFile );
+  }
+  if (false !== (await fsPromise.access(cachedFileMeta).catch(() => false))) {
+    proxyRes = fs.createReadStream(cachedFile);
+    Object.assign( proxyRes, JSON.parse(await fsPromise.readFile(cachedFileMeta)) );
+    isHit = true;
+  } else {
+    proxyRes = await new Promise(res => {
+      const proxyReq = httpModule.request(
+        {
+          host: origUrl.host,
+          port: origUrl.port,
+          path: origUrl.path,
+          method,
+          headers: origReq.headers,
+        },
+        res
+      );
+      origReq.pipe( proxyReq );
+    });
+    await fsPromise.mkdir(dirname(cachedFile), { recursive: true });
+    /**
+     *  write metda data only if the request completed successfully
+     *  Otherwise, partial & invalid cached content will be served next time
+     */
+    origRes.on('finish', () =>  fsPromise.writeFile(cachedFileMeta, JSON.stringify({ headers: proxyRes.headers, statusCode: proxyRes.statusCode } )) )
+    proxyRes.pipe( startNewRequest(cachedFile));
+  }
+
+  console.log(`${new Date().toISOString()} ${isHit ? 'Hit!' : 'Miss'} ${method} ${origReq.url} => ${cachedFile}`);
+
+  origRes.writeHead(proxyRes.statusCode, proxyRes.headers);
+  proxyRes.pipe(origRes);
+  /**
+   *  Don't let download to continue if client closes the connection before it is finished
+   */
+  origRes.on('close', () => proxyRes.destroy() )
+
+  return proxyRes;
+}
+
+
+function createFakeCertificateByDomain(caKey, caCert, domain) {
+  const keys = pki.rsa.generateKeyPair(2048);
+  const cert = pki.createCertificate();
+  cert.publicKey = keys.publicKey;
+
+  cert.serialNumber = new Date().getTime() + '';
+  cert.validity.notBefore = new Date();
+  cert.validity.notBefore.setFullYear(
+    cert.validity.notBefore.getFullYear() - 1
+  );
+  cert.validity.notAfter = new Date();
+  cert.validity.notAfter.setFullYear(cert.validity.notAfter.getFullYear() + 1);
+  var attrs = [
+    {
+      name: 'commonName',
+      value: domain,
+    },
+    {
+      name: 'organizationName',
+      value: 'Proxy-kutti',
+    },
+  ];
+
+  cert.setSubject(attrs);
+  cert.setIssuer(caCert.subject.attributes);
+
+  cert.setExtensions([
+    {
+      name: 'subjectAltName',
+      altNames: [
+        {
+          type: 2,
+          value: domain,
+        },
+      ],
+    },
+    {
+      name: 'extKeyUsage',
+      serverAuth: true,
+    },
+  ]);
+  cert.sign(caKey, forge.md.sha256.create());
+
+  return {
+    key: forge.pki.privateKeyToPem(keys.privateKey),
+    cert: forge.pki.certificateToPem(cert),
+  };
+}
+
+
+function initHttpsMitmProxy() {
+  const caCertPath = config.root_ca_cert;
+  const caKeyPath = config.root_ca_key;
+  const caCertPem = fs.readFileSync(caCertPath);
+  const caKeyPem = fs.readFileSync(caKeyPath);
+  const caCert = forge.pki.certificateFromPem(caCertPem);
+  const caKey = forge.pki.decryptRsaPrivateKey(caKeyPem, 'secret');
+  const fakeCertObj = createFakeCertificateByDomain(caKey, caCert, 'localhost');
+
+  debugger;
+  const https_opts = {
+    key: fakeCertObj.key,
+    cert: fakeCertObj.cert,
+    SNICallback: (hostname, done) => {
+      let certObj = createFakeCertificateByDomain(caKey, caCert, hostname);
+      done(
+        null,
+        tls.createSecureContext({
+          key: certObj.key,
+          cert: certObj.cert,
+        })
+      );
+    },
+  };
+
+
+  const httpsProxy = https.createServer(https_opts, (req, res) => {
+    req.url = `https://${req.headers.host}${req.url}`;
+    getContent(https, req, res);
+  });
+
+  httpsProxy.listen( httpsPort, '127.0.0.1');
+}
+
+
+function main() {
+  const httpProxy = http.createServer(getContent.bind(null, http));
+
+  const isHttpMitmEnabled =
+    fs.existsSync(config.root_ca_cert) && fs.existsSync(config.root_ca_key);
+  let httpsMsg = '';
+  if (isHttpMitmEnabled === false) {
+    httpsMsg = `https requests are not cached since it is not configured.
+  Make sure that the files
+    ${config.root_ca_cert}
+    ${config.root_ca_key}
+  exists and accessible to the process.
+  Refer documentation more details.\n`;
+  } else {
+    initHttpsMitmProxy();
+  }
+
+  const util = require('util');
+  httpProxy.on('connect', function(req, res) {
+    res.write(
+      'HTTP/1.0 200 Connection established\r\nProxy-agent: proxy-kutti\r\n\r\n'
+    );
+    const [host, port] = isHttpMitmEnabled
+      ? ['127.0.0.1',  httpsPort]
+      : req.url.split(':');
+    var httpsProxyConnection = net.createConnection(port, host);
+    res.on('close', () => httpsProxyConnection.destroy());
+    res.pipe(httpsProxyConnection);
+    httpsProxyConnection.pipe(res);
+  });
+
+  httpProxy.listen(config.port, config.host, function() {
+    console.log(`Proxy-kutti is running...
+
+Using env variables
+  PROXY_KUTTI_CONFIG=${configFile}
+
+Current Configuration ( edit ${configFile}.(json|js)  or set env variable PROXY_KUTTI_<config-key>=<value> to change )
+${JSON.stringify(config, null, 2).slice(2, -2)}
+
+${httpsMsg}
+Run the following command shell to start using this proxy
+  export http_proxy=http://${config.host}:${config.port}
+  ${isHttpMitmEnabled? 'export https_proxy=http://'+config.host+':'+config.port: ''}
+
+  `);
+  });
+}
+
+if (require.main === module) {
+  main();
+  process.on('uncaughtException', function (err) {
+    console.log(err);
+  })
+}
--- a/yarn.lock
+++ b/yarn.lock
@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+node-forge@^0.9.1:
+  version "0.9.1"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.1.tgz#775368e6846558ab6676858a4d8c6e8d16c677b5"
+  integrity sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==