index.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433
  1. /*
  2. control character types:
  3. 1 - metadata
  4. 2 - symbols
  5. 6 - false
  6. 7 - true
  7. 8- 16 - negative doubles
  8. 16-24 positive doubles
  9. 27 - String starts with a character 27 or less or is an empty string
  10. 0 - multipart separator
  11. > 27 normal string characters
  12. */
  13. /*
  14. * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
  15. */
  16. const float64Array = new Float64Array(2)
  17. const int32Array = new Int32Array(float64Array.buffer, 0, 4)
  18. let nullTerminate = false
  19. let textEncoder
  20. try {
  21. textEncoder = new TextEncoder()
  22. } catch (error) {}
  23. /*
  24. * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
  25. */
  26. export function writeKey(key, target, position, inSequence) {
  27. let targetView = target.dataView
  28. if (!targetView)
  29. targetView = target.dataView = new DataView(target.buffer, target.byteOffset, ((target.byteLength + 3) >> 2) << 2)
  30. switch (typeof key) {
  31. case 'string':
  32. let strLength = key.length
  33. let c1 = key.charCodeAt(0)
  34. if (!(c1 >= 28)) // escape character
  35. target[position++] = 27
  36. if (strLength < 0x40) {
  37. let i, c2
  38. for (i = 0; i < strLength; i++) {
  39. c1 = key.charCodeAt(i)
  40. if (c1 <= 4) {
  41. target[position++] = 4
  42. target[position++] = c1
  43. } else if (c1 < 0x80) {
  44. target[position++] = c1
  45. } else if (c1 < 0x800) {
  46. target[position++] = c1 >> 6 | 0xc0
  47. target[position++] = c1 & 0x3f | 0x80
  48. } else if (
  49. (c1 & 0xfc00) === 0xd800 &&
  50. ((c2 = key.charCodeAt(i + 1)) & 0xfc00) === 0xdc00
  51. ) {
  52. c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff)
  53. i++
  54. target[position++] = c1 >> 18 | 0xf0
  55. target[position++] = c1 >> 12 & 0x3f | 0x80
  56. target[position++] = c1 >> 6 & 0x3f | 0x80
  57. target[position++] = c1 & 0x3f | 0x80
  58. } else {
  59. target[position++] = c1 >> 12 | 0xe0
  60. target[position++] = c1 >> 6 & 0x3f | 0x80
  61. target[position++] = c1 & 0x3f | 0x80
  62. }
  63. }
  64. } else {
  65. if (target.utf8Write)
  66. position += target.utf8Write(key, position, target.byteLength - position)
  67. else
  68. position += textEncoder.encodeInto(key, target.subarray(position)).written
  69. if (position > target.length - 4)
  70. throw new RangeError('String does not fit in target buffer')
  71. }
  72. break
  73. case 'number':
  74. float64Array[0] = key
  75. let lowInt = int32Array[0]
  76. let highInt = int32Array[1]
  77. let length
  78. if (key < 0) {
  79. targetView.setInt32(position + 4, ~((lowInt >>> 4) | (highInt << 28)))
  80. targetView.setInt32(position + 0, (highInt ^ 0x7fffffff) >>> 4)
  81. targetView.setInt32(position + 8, ((lowInt & 0xf) ^ 0xf) << 4, true) // just always do the null termination here
  82. return position + 9
  83. } else if ((lowInt & 0xf) || inSequence) {
  84. length = 9
  85. } else if (lowInt & 0xfffff)
  86. length = 8
  87. else if (lowInt || (highInt & 0xf))
  88. length = 6
  89. else
  90. length = 4
  91. // switching order to go to little endian
  92. targetView.setInt32(position + 0, (highInt >>> 4) | 0x10000000)
  93. targetView.setInt32(position + 4, (lowInt >>> 4) | (highInt << 28))
  94. // if (length == 9 || nullTerminate)
  95. targetView.setInt32(position + 8, (lowInt & 0xf) << 4, true)
  96. return position + length;
  97. case 'object':
  98. if (key) {
  99. if (Array.isArray(key)) {
  100. for (let i = 0, l = key.length; i < l; i++) {
  101. if (i > 0)
  102. target[position++] = 0
  103. position = writeKey(key[i], target, position, true)
  104. }
  105. break
  106. } else if (key instanceof Uint8Array) {
  107. target.set(key, position)
  108. position += key.length
  109. break
  110. } else {
  111. throw new Error('Unable to serialize object as a key: ' + JSON.stringify(key))
  112. }
  113. } else // null
  114. target[position++] = 0
  115. break
  116. case 'boolean':
  117. targetView.setUint32(position++, key ? 7 : 6, true)
  118. return position
  119. case 'bigint':
  120. let asFloat = Number(key)
  121. if (BigInt(asFloat) > key) {
  122. float64Array[0] = asFloat;
  123. if (asFloat > 0) {
  124. if (int32Array[0])
  125. int32Array[0]--;
  126. else {
  127. int32Array[1]--;
  128. int32Array[0] = 0xffffffff;
  129. }
  130. } else {
  131. if (int32Array[0] < 0xffffffff)
  132. int32Array[0]++;
  133. else {
  134. int32Array[1]++;
  135. int32Array[0] = 0;
  136. }
  137. }
  138. asFloat = float64Array[0];
  139. }
  140. let difference = key - BigInt(asFloat);
  141. if (difference === 0n)
  142. return writeKey(asFloat, target, position, inSequence)
  143. writeKey(asFloat, target, position, inSequence)
  144. position += 9; // always increment by 9 if we are adding fractional bits
  145. let exponent = BigInt((int32Array[1] >> 20 & 0x7ff) - 1079);
  146. let nextByte = difference >> exponent;
  147. target[position - 1] |= Number(nextByte);
  148. difference -= nextByte << exponent;
  149. let first = true;
  150. while (difference || first) {
  151. first = false;
  152. exponent -= 7n;
  153. let nextByte = difference >> exponent;
  154. target[position++] = Number(nextByte) | 0x80;
  155. difference -= nextByte << exponent;
  156. }
  157. return position;
  158. case 'undefined':
  159. return position
  160. // undefined is interpreted as the absence of a key, signified by zero length
  161. case 'symbol':
  162. target[position++] = 2
  163. return writeKey(key.description, target, position, inSequence)
  164. default:
  165. throw new Error('Can not serialize key of type ' + typeof key)
  166. }
  167. if (nullTerminate && !inSequence)
  168. targetView.setUint32(position, 0)
  169. return position
  170. }
  171. let position
  172. export function readKey(buffer, start, end, inSequence) {
  173. position = start
  174. let controlByte = buffer[position]
  175. let value
  176. if (controlByte < 24) {
  177. if (controlByte < 8) {
  178. position++
  179. if (controlByte == 6) {
  180. value = false
  181. } else if (controlByte == 7) {
  182. value = true
  183. } else if (controlByte == 0) {
  184. value = null
  185. } else if (controlByte == 2) {
  186. value = Symbol.for(readStringSafely(buffer, end))
  187. } else
  188. return Uint8Array.prototype.slice.call(buffer, start, end)
  189. } else {
  190. let dataView;
  191. try {
  192. dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2))
  193. } catch(error) {
  194. // if it is write at the end of the ArrayBuffer, we may need to retry with the exact remaining bytes
  195. dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.buffer.byteLength - buffer.byteOffset))
  196. }
  197. let highInt = dataView.getInt32(position) << 4
  198. let size = end - position
  199. let lowInt
  200. if (size > 4) {
  201. lowInt = dataView.getInt32(position + 4)
  202. highInt |= lowInt >>> 28
  203. if (size <= 6) { // clear the last bits
  204. lowInt &= -0x10000
  205. }
  206. lowInt = lowInt << 4
  207. if (size > 8) {
  208. lowInt = lowInt | buffer[position + 8] >> 4
  209. }
  210. } else
  211. lowInt = 0
  212. if (controlByte < 16) {
  213. // negative gets negated
  214. highInt = highInt ^ 0x7fffffff
  215. lowInt = ~lowInt
  216. }
  217. int32Array[1] = highInt
  218. int32Array[0] = lowInt
  219. value = float64Array[0]
  220. position += 9
  221. if (size > 9 && buffer[position] > 0) {
  222. // convert the float to bigint, and then we will add precision as we enumerate through the
  223. // extra bytes
  224. value = BigInt(value);
  225. let exponent = highInt >> 20 & 0x7ff;
  226. let next_byte = buffer[position - 1] & 0xf;
  227. value += BigInt(next_byte) << BigInt(exponent - 1079);
  228. while ((next_byte = buffer[position]) > 0 && position++ < end) {
  229. value += BigInt(next_byte & 0x7f) << BigInt((start - position) * 7 + exponent - 1016);
  230. }
  231. }
  232. }
  233. } else {
  234. if (controlByte == 27) {
  235. position++
  236. }
  237. value = readStringSafely(buffer, end)
  238. }
  239. while (position < end) {
  240. if (buffer[position] === 0)
  241. position++
  242. if (inSequence) {
  243. encoder.position = position
  244. return value
  245. }
  246. let nextValue = readKey(buffer, position, end, true)
  247. if (value instanceof Array) {
  248. value.push(nextValue)
  249. } else
  250. value = [ value, nextValue ]
  251. }
  252. return value
  253. }
  254. export const enableNullTermination = () => nullTerminate = true
  255. export const encoder = {
  256. writeKey,
  257. readKey,
  258. enableNullTermination,
  259. }
  260. let targetBuffer = []
  261. let targetPosition = 0
  262. const hasNodeBuffer = typeof Buffer !== 'undefined'
  263. const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array
  264. export const toBufferKey = (key) => {
  265. let newBuffer
  266. if (targetPosition + 100 > targetBuffer.length) {
  267. targetBuffer = new ByteArrayAllocate(8192)
  268. targetPosition = 0
  269. newBuffer = true
  270. }
  271. try {
  272. let result = targetBuffer.slice(targetPosition, targetPosition = writeKey(key, targetBuffer, targetPosition))
  273. if (targetPosition > targetBuffer.length) {
  274. if (newBuffer)
  275. throw new Error('Key is too large')
  276. return toBufferKey(key)
  277. }
  278. return result
  279. } catch(error) {
  280. if (newBuffer)
  281. throw error
  282. targetPosition = targetBuffer.length
  283. return toBufferKey(key)
  284. }
  285. }
  286. export const fromBufferKey = (sourceBuffer) => {
  287. return readKey(sourceBuffer, 0, sourceBuffer.length)
  288. }
  289. const fromCharCode = String.fromCharCode
  290. function makeStringBuilder() {
  291. let stringBuildCode = '(source) => {'
  292. let previous = []
  293. for (let i = 0; i < 0x30; i++) {
  294. let v = fromCharCode((i & 0xf) + 97) + fromCharCode((i >> 4) + 97)
  295. stringBuildCode += `
  296. let ${v} = source[position++]
  297. if (${v} > 4) {
  298. if (${v} >= 0x80) ${v} = finishUtf8(${v}, source)
  299. } else {
  300. if (${v} === 4)
  301. ${v} = source[position++]
  302. else
  303. return fromCharCode(${previous})
  304. }
  305. `
  306. previous.push(v)
  307. if (i == 1000000) // this just exists to prevent rollup from doing dead code elimination on finishUtf8
  308. finishUtf8()
  309. }
  310. stringBuildCode += `return fromCharCode(${previous}) + readString(source)}`
  311. return stringBuildCode
  312. }
  313. let pendingSurrogate
  314. function finishUtf8(byte1, src) {
  315. if ((byte1 & 0xe0) === 0xc0) {
  316. // 2 bytes
  317. const byte2 = src[position++] & 0x3f
  318. return ((byte1 & 0x1f) << 6) | byte2
  319. } else if ((byte1 & 0xf0) === 0xe0) {
  320. // 3 bytes
  321. const byte2 = src[position++] & 0x3f
  322. const byte3 = src[position++] & 0x3f
  323. return ((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3
  324. } else if ((byte1 & 0xf8) === 0xf0) {
  325. // 4 bytes
  326. if (pendingSurrogate) {
  327. byte1 = pendingSurrogate
  328. pendingSurrogate = null
  329. position += 3
  330. return byte1
  331. }
  332. const byte2 = src[position++] & 0x3f
  333. const byte3 = src[position++] & 0x3f
  334. const byte4 = src[position++] & 0x3f
  335. let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4
  336. if (unit > 0xffff) {
  337. pendingSurrogate = 0xdc00 | (unit & 0x3ff)
  338. unit = (((unit - 0x10000) >>> 10) & 0x3ff) | 0xd800
  339. position -= 4 // reset so we can return the next part of the surrogate pair
  340. }
  341. return unit
  342. } else {
  343. return byte1
  344. }
  345. }
  346. const readString =
  347. typeof process !== 'undefined' && process.isBun ? // the eval in bun doesn't properly closure on position, so we
  348. // have to manually update it
  349. (function(reading) {
  350. let { setPosition, getPosition, readString } = reading;
  351. return (source) => {
  352. setPosition(position);
  353. let value = readString(source);
  354. position = getPosition();
  355. return value;
  356. };
  357. })((new Function('fromCharCode', 'let position; let readString = ' + makeStringBuilder() +
  358. ';return {' +
  359. 'setPosition(p) { position = p },' +
  360. 'getPosition() { return position },' +
  361. 'readString }'))(fromCharCode)) :
  362. eval(makeStringBuilder())
  363. function readStringSafely(source, end) {
  364. if (source[end] > 0) {
  365. let previous = source[end]
  366. try {
  367. // read string expects a null terminator, that is a 0 or undefined from reading past the end of the buffer, so we
  368. // have to ensure that, but do so safely, restoring the buffer to its original state
  369. source[end] = 0
  370. return readString(source)
  371. } finally {
  372. source[end] = previous
  373. }
  374. } else return readString(source);
  375. }
  376. export function compareKeys(a, b) {
  377. // compare with type consistency that matches binary comparison
  378. if (typeof a == 'object') {
  379. if (!a) {
  380. return b == null ? 0 : -1
  381. }
  382. if (a.compare) {
  383. if (b == null) {
  384. return 1
  385. } else if (b.compare) {
  386. return a.compare(b)
  387. } else {
  388. return -1
  389. }
  390. }
  391. let arrayComparison
  392. if (b instanceof Array) {
  393. let i = 0
  394. while((arrayComparison = compareKeys(a[i], b[i])) == 0 && i <= a.length) {
  395. i++
  396. }
  397. return arrayComparison
  398. }
  399. arrayComparison = compareKeys(a[0], b)
  400. if (arrayComparison == 0 && a.length > 1)
  401. return 1
  402. return arrayComparison
  403. } else if (typeof a == typeof b) {
  404. if (typeof a === 'symbol') {
  405. a = Symbol.keyFor(a)
  406. b = Symbol.keyFor(b)
  407. }
  408. return a < b ? -1 : a === b ? 0 : 1
  409. }
  410. else if (typeof b == 'object') {
  411. if (b instanceof Array)
  412. return -compareKeys(b, a)
  413. return 1
  414. } else {
  415. return typeOrder[typeof a] < typeOrder[typeof b] ? -1 : 1
  416. }
  417. }
  418. const typeOrder = {
  419. symbol: 0,
  420. undefined: 1,
  421. boolean: 2,
  422. number: 3,
  423. string: 4
  424. }
  425. export const MINIMUM_KEY = null
  426. export const MAXIMUM_KEY = new Uint8Array([0xff])