Skip to content
Snippets Groups Projects
Unverified Commit 86735d75 authored by Sergei Zharinov's avatar Sergei Zharinov Committed by GitHub
Browse files

refactor(rubygems): Extract `copystring` function to the utils (#22322)

parent 43669f86
No related merge requests found
...@@ -6,6 +6,7 @@ import { getElapsedMinutes } from '../../../util/date'; ...@@ -6,6 +6,7 @@ import { getElapsedMinutes } from '../../../util/date';
import { HttpError } from '../../../util/http'; import { HttpError } from '../../../util/http';
import { newlineRegex } from '../../../util/regex'; import { newlineRegex } from '../../../util/regex';
import { LooseArray } from '../../../util/schema-utils'; import { LooseArray } from '../../../util/schema-utils';
import { copystr } from '../../../util/string';
import { Datasource } from '../datasource'; import { Datasource } from '../datasource';
import type { GetReleasesConfig, ReleaseResult } from '../types'; import type { GetReleasesConfig, ReleaseResult } from '../types';
...@@ -51,8 +52,6 @@ const Lines = z ...@@ -51,8 +52,6 @@ const Lines = z
type Lines = z.infer<typeof Lines>; type Lines = z.infer<typeof Lines>;
export class VersionsDatasource extends Datasource { export class VersionsDatasource extends Datasource {
private isInitialFetch = true;
constructor(override readonly id: string) { constructor(override readonly id: string) {
super(id); super(id);
} }
...@@ -97,31 +96,12 @@ export class VersionsDatasource extends Datasource { ...@@ -97,31 +96,12 @@ export class VersionsDatasource extends Datasource {
return { releases }; return { releases };
} }
/**
* Since each `/versions` reponse exceed 10MB,
* there is potential for a memory leak if we construct slices
* of the response body and cache them long-term:
*
* https://bugs.chromium.org/p/v8/issues/detail?id=2869
*
* This method meant to be called for `version` and `packageName`
* before storing them in the cache.
*/
private copystr(x: string): string {
const len = Buffer.byteLength(x, 'utf8');
const buf = this.isInitialFetch
? Buffer.allocUnsafe(len) // allocate from pre-allocated buffer
: Buffer.allocUnsafeSlow(len); // allocate standalone buffer
buf.write(x, 'utf8');
return buf.toString('utf8');
}
private updatePackageReleases( private updatePackageReleases(
packageReleases: PackageReleases, packageReleases: PackageReleases,
lines: Lines lines: Lines
): void { ): void {
for (const line of lines) { for (const line of lines) {
const packageName = this.copystr(line.packageName); const packageName = copystr(line.packageName);
let versions = packageReleases.get(packageName) ?? []; let versions = packageReleases.get(packageName) ?? [];
const { deletedVersions, addedVersions } = line; const { deletedVersions, addedVersions } = line;
...@@ -134,7 +114,7 @@ export class VersionsDatasource extends Datasource { ...@@ -134,7 +114,7 @@ export class VersionsDatasource extends Datasource {
const existingVersions = new Set(versions); const existingVersions = new Set(versions);
for (const addedVersion of addedVersions) { for (const addedVersion of addedVersions) {
if (!existingVersions.has(addedVersion)) { if (!existingVersions.has(addedVersion)) {
const version = this.copystr(addedVersion); const version = copystr(addedVersion);
versions.push(version); versions.push(version);
} }
} }
...@@ -183,7 +163,6 @@ export class VersionsDatasource extends Datasource { ...@@ -183,7 +163,6 @@ export class VersionsDatasource extends Datasource {
const lines = Lines.parse(newLines); const lines = Lines.parse(newLines);
this.updatePackageReleases(regCache.packageReleases, lines); this.updatePackageReleases(regCache.packageReleases, lines);
this.isInitialFetch = false;
} }
private updateRubyGemsVersionsPromise: Promise<void> | null = null; private updateRubyGemsVersionsPromise: Promise<void> | null = null;
......
...@@ -67,3 +67,18 @@ export function titleCase(input: string): string { ...@@ -67,3 +67,18 @@ export function titleCase(input: string): string {
return words.join(' '); return words.join(' ');
} }
/**
* Sometimes we extract small strings from a multi-megabyte files.
* If we then save them in the in-memory cache, V8 may not free
* the initial buffer, which can lead to memory leaks:
*
* https://bugs.chromium.org/p/v8/issues/detail?id=2869
*
*/
export function copystr(x: string): string {
const len = Buffer.byteLength(x, 'utf8');
const buf = Buffer.allocUnsafeSlow(len);
buf.write(x, 'utf8');
return buf.toString('utf8');
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment