8000 Performance improvements by oschwald · Pull Request #225 · runk/mmdb-lib · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Performance improvements #225

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Apr 29, 2025
Merged

Performance improvements #225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dbab38f
Do not create extra array when decoding string
oschwald Apr 29, 2025
640bc73
Pre-allocate array rather than growing it
oschwald Apr 29, 2025
5386bf8
Replace deprecated slice with subarray
oschwald Apr 29, 2025
1e8385f
Use readUInt*BE functions
oschwald Apr 29, 2025
1218d66
Correct handling of decodeInt for size != 4
oschwald Apr 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions src/decoder.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,30 @@ describe('lib/decoder', () => {
});
});

describe('decodeInt()', () => {
const testCases = [
{ expected: 0, input: [0x0, 0x1] },
{ expected: -1, input: [0x4, 0x1, 0xff, 0xff, 0xff, 0xff] },
{ expected: 255, input: [0x1, 0x1, 0xff] },
{ expected: -255, input: [0x4, 0x1, 0xff, 0xff, 0xff, 0x1] },
{ expected: 500, input: [0x2, 0x1, 0x1, 0xf4] },
{ expected: -500, input: [0x4, 0x1, 0xff, 0xff, 0xfe, 0xc] },
{ expected: 65535, input: [0x2, 0x1, 0xff, 0xff] },
{ expected: -65535, input: [0x4, 0x1, 0xff, 0xff, 0x0, 0x1] },
{ expected: 16777215, input: [0x3, 0x1, 0xff, 0xff, 0xff] },
{ expected: -16777215, input: [0x4, 0x1, 0xff, 0x0, 0x0, 0x1] },
{ expected: 2147483647, input: [0x4, 0x1, 0x7f, 0xff, 0xff, 0xff] },
{ expected: -2147483647, input: [0x4, 0x1, 0x80, 0x0, 0x0, 0x1] },
];

for (let tc of testCases) {
it(`should decode to ${tc.expected}`, () => {
const decoder = new Decoder(Buffer.from(tc.input));
assert.deepStrictEqual(decoder.decode(0).value, tc.expected);
});
}
});

describe('decode()', () => {
it('should throw when extended type has wrong size', () => {
const test = new Decoder(Buffer.from([0x00, 0x00]));
Expand Down
73 changes: 23 additions & 50 deletions src/decoder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,19 +169,11 @@ export default class Decoder {
// At this point `size` is always 31.
// If the value is 31, then the size is 65,821 + *the next three bytes after the
// type specifying bytes as a single unsigned integer*.
return cursor(
65821 +
utils.concat3(
this.db[offset],
this.db[offset + 1],
this.db[offset + 2]
),
offset + 3
);
return cursor(65821 + this.db.readUIntBE(offset, 3), offset + 3);
}

private decodeBytes(offset: number, size: number): Buffer {
return this.db.slice(offset, offset + size);
return this.db.subarray(offset, offset + size);
}

private decodePointer(ctrlByte: number, offset: number): Cursor {
Expand All @@ -201,26 +193,17 @@ e 10000 xport default class Decoder {
// If the size is 0, the pointer is built by appending the next byte to the last
// three bits to produce an 11-bit value.
if (pointerSize === 0) {
packed = utils.concat2(ctrlByte & 7, this.db[offset]);
packed = ((ctrlByte & 7) << 8) | this.db[offset];

// If the size is 1, the pointer is built by appending the next two bytes to the
// last three bits to produce a 19-bit value + 2048.
} else if (pointerSize === 1) {
packed = utils.concat3(
ctrlByte & 7,
this.db[offset],
this.db[offset + 1]
);
packed = ((ctrlByte & 7) << 16) | this.db.readUInt16BE(offset);

// If the size is 2, the pointer is built by appending the next three bytes to the
// last three bits to produce a 27-bit value + 526336.
} else if (pointerSize === 2) {
packed = utils.concat4(
ctrlByte & 7,
this.db[offset],
this.db[offset + 1],
this.db[offset + 2]
);
packed = ((ctrlByte & 7) << 24) | this.db.readUIntBE(offset, 3);

// At next point `size` is always 3.
// Finally, if the size is 3, the pointer's value is contained in the next four
Expand All @@ -236,12 +219,12 @@ export default class Decoder {

private decodeArray(size: number, offset: number): Cursor {
let tmp;
const array = [];
const array = new Array(size);

for (let i = 0; i < size; i++) {
tmp = this.decode(offset);
offset = tmp.offset;
array.push(tmp.value);
array[i] = tmp.value;
}

return cursor(array, offset);
Expand Down Expand Up @@ -280,40 +263,30 @@ export default class Decoder {
if (size === 0) {
return 0;
}
if (size < 4) {
return this.db.readUIntBE(offset, size);
}
return this.db.readInt32BE(offset);
}

private decodeUint(offset: number, size: number) {
switch (size) {
case 0:
return 0;
case 1:
return this.db[offset];
case 2:
return utils.concat2(this.db[offset + 0], this.db[offset + 1]);
case 3:
return utils.concat3(
this.db[offset + 0],
this.db[offset + 1],
this.db[offset + 2]
);
case 4:
return utils.concat4(
this.db[offset + 0],
this.db[offset + 1],
this.db[offset + 2],
this.db[offset + 3]
);
case 8:
return this.decodeBigUint(offset, size);
case 16:
return this.decodeBigUint(offset, size);
if (size === 0) {
return 0;
}
if (size <= 6) {
return this.db.readUIntBE(offset, size);
}
if (size == 8) {
return this.db.readBigInt64BE(offset).toString();
}
if (size > 16) {
return 0;
}
return 0;
return this.decodeBigUint(offset, size);
}

private decodeString(offset: number, size: number) {
return this.db.slice(offset, offset + size).toString();
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice. Not sure why I did it that way in first place.

return this.db.toString('utf8', offset, offset + size);
}

private decodeBigUint(offset: number, size: number) {
Expand Down
20 changes: 4 additions & 16 deletions src/reader/walker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import utils from '../utils';

type NodeReader = (offset: number) => number;

export interface Walker {
Expand All @@ -10,32 +8,22 @@ export interface Walker {
const readNodeRight24 =
(db: Buffer): NodeReader =>
(offset: number): number =>
utils.concat3(db[offset + 3], db[offset + 4], db[offset + 5]);
db.readUIntBE(offset + 3, 3);

const readNodeLeft24 =
(db: Buffer): NodeReader =>
(offset: number): number =>
utils.concat3(db[offset], db[offset + 1], db[offset + 2]);
db.readUIntBE(offset, 3);

const readNodeLeft28 =
(db: Buffer): NodeReader =>
(offset: number): number =>
utils.concat4(
db[offset + 3] >> 4,
db[offset],
db[offset + 1],
db[offset + 2]
);
((db[offset + 3] & 0xf0) << 20) | db.readUIntBE(offset, 3);

const readNodeRight28 =
(db: Buffer): NodeReader =>
(offset: number): number =>
utils.concat4(
db[offset + 3] & 0x0f,
db[offset + 4],
db[offset + 5],
db[offset + 6]
);
((db[offset + 3] & 0x0f) << 24) | db.readUIntBE(offset + 4, 3);

const readNodeLeft32 =
(db: Buffer): NodeReader =>
Expand Down
15 changes: 0 additions & 15 deletions src/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,3 @@
const concat2 = (a: number, b: number): number => {
return (a << 8) | b;
};

const concat3 = (a: number, b: number, c: number): number => {
return (a << 16) | (b << 8) | c;
};

const concat4 = (a: number, b: number, c: number, d: number): number => {
return (a << 24) | (b << 16) | (c << 8) | d;
};

const legacyErrorMessage = `Maxmind v2 module has changed API.\n\
Upgrade instructions can be found here: \
https://github.com/runk/node-maxmind/wiki/Migration-guide\n\
Expand All @@ -23,8 +11,5 @@ const assert = (condition: boolean, message: string): void => {

export default {
assert,
concat2,
concat3,
concat4,
legacyErrorMessage,
};
0