Skip to content
Snippets Groups Projects
Unverified Commit bd0adf2d authored by Sergei Zharinov's avatar Sergei Zharinov Committed by GitHub
Browse files

refactor: Deprecate compression with `base64` encoding (#26711)

parent a055853b
No related branches found
No related tags found
No related merge requests found
...@@ -2,7 +2,7 @@ import cacache from 'cacache'; ...@@ -2,7 +2,7 @@ import cacache from 'cacache';
import { DateTime } from 'luxon'; import { DateTime } from 'luxon';
import upath from 'upath'; import upath from 'upath';
import { logger } from '../../../logger'; import { logger } from '../../../logger';
import { compress, decompress } from '../../compress'; import { compressToBase64, decompressFromBase64 } from '../../compress';
function getKey(namespace: string, key: string): string { function getKey(namespace: string, key: string): string {
return `${namespace}-${key}`; return `${namespace}-${key}`;
...@@ -32,7 +32,7 @@ export async function get<T = never>( ...@@ -32,7 +32,7 @@ export async function get<T = never>(
if (!cachedValue.compress) { if (!cachedValue.compress) {
return cachedValue.value; return cachedValue.value;
} }
const res = await decompress(cachedValue.value); const res = await decompressFromBase64(cachedValue.value);
return JSON.parse(res); return JSON.parse(res);
} }
await rm(namespace, key); await rm(namespace, key);
...@@ -58,7 +58,7 @@ export async function set( ...@@ -58,7 +58,7 @@ export async function set(
getKey(namespace, key), getKey(namespace, key),
JSON.stringify({ JSON.stringify({
compress: true, compress: true,
value: await compress(JSON.stringify(value)), value: await compressToBase64(value),
expiry: DateTime.local().plus({ minutes: ttlMinutes }), expiry: DateTime.local().plus({ minutes: ttlMinutes }),
}), }),
); );
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
import { DateTime } from 'luxon'; import { DateTime } from 'luxon';
import { createClient } from 'redis'; import { createClient } from 'redis';
import { logger } from '../../../logger'; import { logger } from '../../../logger';
import { compress, decompress } from '../../compress'; import { compressToBase64, decompressFromBase64 } from '../../compress';
let client: ReturnType<typeof createClient> | undefined; let client: ReturnType<typeof createClient> | undefined;
let rprefix: string | undefined; let rprefix: string | undefined;
...@@ -43,7 +43,7 @@ export async function get<T = never>( ...@@ -43,7 +43,7 @@ export async function get<T = never>(
if (!cachedValue.compress) { if (!cachedValue.compress) {
return cachedValue.value; return cachedValue.value;
} }
const res = await decompress(cachedValue.value); const res = await decompressFromBase64(cachedValue.value);
return JSON.parse(res); return JSON.parse(res);
} }
// istanbul ignore next // istanbul ignore next
...@@ -71,7 +71,7 @@ export async function set( ...@@ -71,7 +71,7 @@ export async function set(
getKey(namespace, key), getKey(namespace, key),
JSON.stringify({ JSON.stringify({
compress: true, compress: true,
value: await compress(JSON.stringify(value)), value: await compressToBase64(value),
expiry: DateTime.local().plus({ minutes: ttlMinutes }), expiry: DateTime.local().plus({ minutes: ttlMinutes }),
}), }),
{ EX: redisTTL }, { EX: redisTTL },
......
import is from '@sindresorhus/is'; import is from '@sindresorhus/is';
import { GlobalConfig } from '../../../../config/global'; import { GlobalConfig } from '../../../../config/global';
import { logger } from '../../../../logger'; import { logger } from '../../../../logger';
import { compress, decompress } from '../../../compress'; import { compressToBase64, decompressFromBase64 } from '../../../compress';
import { hash } from '../../../hash'; import { hash } from '../../../hash';
import { safeStringify } from '../../../stringify'; import { safeStringify } from '../../../stringify';
import { CACHE_REVISION } from '../common'; import { CACHE_REVISION } from '../common';
...@@ -41,7 +41,7 @@ export abstract class RepoCacheBase implements RepoCache { ...@@ -41,7 +41,7 @@ export abstract class RepoCacheBase implements RepoCache {
logger.debug('Repository cache fingerprint is invalid'); logger.debug('Repository cache fingerprint is invalid');
return; return;
} }
const jsonStr = await decompress(oldCache.payload); const jsonStr = await decompressFromBase64(oldCache.payload);
this.data = RepoCacheBase.parseData(jsonStr); this.data = RepoCacheBase.parseData(jsonStr);
this.oldHash = oldCache.hash; this.oldHash = oldCache.hash;
} }
...@@ -80,7 +80,7 @@ export abstract class RepoCacheBase implements RepoCache { ...@@ -80,7 +80,7 @@ export abstract class RepoCacheBase implements RepoCache {
const repository = this.repository; const repository = this.repository;
const fingerprint = this.fingerprint; const fingerprint = this.fingerprint;
const payload = await compress(jsonStr); const payload = await compressToBase64(jsonStr);
await this.write({ await this.write({
revision, revision,
......
import { fs } from '../../../../../test/util'; import { fs } from '../../../../../test/util';
import { GlobalConfig } from '../../../../config/global'; import { GlobalConfig } from '../../../../config/global';
import { logger } from '../../../../logger'; import { logger } from '../../../../logger';
import { compress } from '../../../compress'; import { compressToBase64 } from '../../../compress';
import { hash } from '../../../hash'; import { hash } from '../../../hash';
import { CACHE_REVISION } from '../common'; import { CACHE_REVISION } from '../common';
import type { RepoCacheRecord } from '../schema'; import type { RepoCacheRecord } from '../schema';
...@@ -21,7 +21,7 @@ async function createCacheRecord( ...@@ -21,7 +21,7 @@ async function createCacheRecord(
const jsonStr = JSON.stringify(data); const jsonStr = JSON.stringify(data);
const hashedJsonStr = hash(jsonStr); const hashedJsonStr = hash(jsonStr);
const payload = await compress(jsonStr); const payload = await compressToBase64(jsonStr);
return { return {
revision, revision,
......
import { compress, decompress } from './compress'; import { compressToBase64, decompressFromBase64 } from './compress';
describe('util/compress', () => { describe('util/compress', () => {
it('works', async () => { it('compresses strings', async () => {
const input = 'foobar'; const input = 'foobar';
const compressed = await compress(input); const compressed = await compressToBase64(input);
expect(compressed).toBe('iwKAZm9vYmFyAw=='); expect(compressed).toBe('iwKAZm9vYmFyAw==');
const decompressed = await decompress(compressed); const decompressed = await decompressFromBase64(compressed);
expect(decompressed).toBe(input); expect(decompressed).toBe(input);
}); });
it('compresses objects', async () => {
const input = { foo: 'bar' };
const compressed = await compressToBase64(input);
expect(compressed).toBe('CwaAeyJmb28iOiJiYXIifQM=');
const decompressed = await decompressFromBase64(compressed);
expect(JSON.parse(decompressed)).toEqual(input);
});
}); });
import { promisify } from 'node:util'; import { promisify } from 'node:util';
import zlib, { constants } from 'node:zlib'; import zlib, { constants } from 'node:zlib';
import is from '@sindresorhus/is';
const brotliCompress = promisify(zlib.brotliCompress); const brotliCompress = promisify(zlib.brotliCompress);
const brotliDecompress = promisify(zlib.brotliDecompress); const brotliDecompress = promisify(zlib.brotliDecompress);
export async function compress(input: string): Promise<string> { /**
const buf = await brotliCompress(input, { * @deprecated
*/
export async function compressToBase64(input: unknown): Promise<string> {
const jsonStr = is.string(input) ? input : JSON.stringify(input);
const buf = await brotliCompress(jsonStr, {
params: { params: {
[constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT, [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT,
[constants.BROTLI_PARAM_QUALITY]: 8, [constants.BROTLI_PARAM_QUALITY]: 8,
...@@ -14,7 +19,10 @@ export async function compress(input: string): Promise<string> { ...@@ -14,7 +19,10 @@ export async function compress(input: string): Promise<string> {
return buf.toString('base64'); return buf.toString('base64');
} }
export async function decompress(input: string): Promise<string> { /**
* @deprecated
*/
export async function decompressFromBase64(input: string): Promise<string> {
const buf = Buffer.from(input, 'base64'); const buf = Buffer.from(input, 'base64');
const str = await brotliDecompress(buf); const str = await brotliDecompress(buf);
return str.toString('utf8'); return str.toString('utf8');
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment