1
0
mirror of https://github.com/qntm/base65536 synced 2026-03-27 01:34:45 +01:00

Compare commits

..

No commits in common. "4add117c1359abcaeac8d514c7b9162b7f008547" and "1ef6e737f4def274f0edde6948a1dee03e8dceb7" have entirely different histories.

18 changed files with 141 additions and 8740 deletions

View File

@ -1,7 +1,7 @@
[*] [*]
insert_final_newline = true insert_final_newline = true
[{*.js,*.json,*.ts,*.md,*.html}] [{*.js,*.json,*.ts,*.md}]
charset = utf-8 charset = utf-8
indent_style = space indent_style = space
indent_size = 2 indent_size = 2

2
.gitignore vendored
View File

@ -46,4 +46,4 @@ Network Trash Folder
Temporary Items Temporary Items
.apdisk .apdisk
node_modules node_modules
coverage dist

View File

@ -1,21 +0,0 @@
# CHANGELOG
## 4.x.x
`base65536` now ships as JavaScript ("ES6") modules, not CommonJS modules. The package has `"type": "module"`. Additionally a separate IIFE for browser use is no longer distributed.
## 3.x.x
`encode` and `decode` now use `Uint8Arrays`, not `ArrayBuffer`s. The options `ignoreGarbage` and `wrap` have been scrapped.
## 2.x.x
`encode` and `decode` now operate on environment-agnostic `ArrayBuffer`s instead of Node.js-specific `Buffer`s.
## 1.x.x
No breaking changes; in this version, the code point repertoire was deemed final.
## 0.0.x
Prototype release.

View File

@ -136,6 +136,17 @@ console.log(uint8Array2);
// [104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100] // [104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100]
``` ```
### In the browser
Load this file in the browser to gain access to a `base65536` global.
```html
<script src='https://unpkg.com/base65536@3/dist/iife/base65536.js' crossorigin></script>
<script>
console.log(base65536.decode('驨ꍬ啯𒁷ꍲᕤ'))
</script>
```
## API ## API
`base65536` accepts and returns `Uint8Array`s. Note that every Node.js `Buffer` is a `Uint8Array`. A `Uint8Array` can be converted to a Node.js `Buffer` like so: `base65536` accepts and returns `Uint8Array`s. Note that every Node.js `Buffer` is a `Uint8Array`. A `Uint8Array` can be converted to a Node.js `Buffer` like so:

View File

@ -1,5 +1,6 @@
{ {
"fixturesFolder": false, "fixturesFolder": false,
"integrationFolder": "test-built",
"pluginsFile": false, "pluginsFile": false,
"screenshotOnRunFailure": false, "screenshotOnRunFailure": false,
"supportFile": false, "supportFile": false,

View File

@ -1,8 +0,0 @@
/* global describe, it, cy */
describe('base65536 in the browser', () => {
it('works', () => {
cy.visit('http://localhost:3000/server/index.html')
cy.contains('what the heck is up')
})
})

8645
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,24 +1,28 @@
{ {
"name": "base65536", "name": "base65536",
"version": "4.0.1", "version": "3.0.3",
"description": "Unicode's answer to Base64", "description": "Unicode's answer to Base64",
"homepage": "https://github.com/qntm/base65536", "homepage": "https://github.com/qntm/base65536",
"repository": { "repository": {
"type": "git", "type": "git",
"url": "git://github.com/qntm/base65536.git" "url": "git://github.com/qntm/base65536.git"
}, },
"type": "module", "module": "dist/es6/base65536.js",
"main": "src", "main": "dist/cjs/base65536.js",
"types": "typings/index.d.ts", "types": "typings/index.d.ts",
"scripts": { "scripts": {
"cypress": "start-server-and-test \"node server/server.js\" http://localhost:3000/server/index.html \"cypress run\"", "build": "rollup -c rollup.config.js",
"postpublish": "npm version patch && git push", "jest": "jest",
"jest": "cross-env NODE_OPTIONS=--experimental-vm-modules npx jest --coverage --transform {} --testPathIgnorePatterns cypress",
"standard": "standard", "standard": "standard",
"test": "npm run standard && npm run jest && npm run cypress" "test": "standard && jest",
"test-built": "npm run test-built-cjs && npm run test-built-es6 && npm run test-built-iife",
"test-built-cjs": "node test-built/cjs.js",
"test-built-es6": "babel-node test-built/es6.js",
"test-built-iife": "start-server-and-test \"node test-built/server.js\" http://localhost:3000/test-built/index.html \"cypress run --spec test-built/iife.js\""
}, },
"keywords": [ "keywords": [
"base64", "base64",
"base65536",
"encoding", "encoding",
"unicode", "unicode",
"text", "text",
@ -28,16 +32,15 @@
"author": "qntm", "author": "qntm",
"license": "MIT", "license": "MIT",
"devDependencies": { "devDependencies": {
"@babel/eslint-parser": "^7.15.0",
"@babel/node": "^7.10.5", "@babel/node": "^7.10.5",
"@babel/preset-env": "^7.8.4", "@babel/preset-env": "^7.8.4",
"base65536-test": "^1.1.2", "base65536-test": "^1.1.2",
"cross-env": "^7.0.3",
"cypress": "^8.3.1", "cypress": "^8.3.1",
"express": "^4.17.1", "express": "^4.17.1",
"glob": "^7.1.6", "glob": "^7.1.6",
"jest": "^27.0.3", "jest": "^27.0.3",
"safe-code-point": "^3.0.0", "rollup": "^2.26.3",
"safe-code-point": "^2.0.0",
"standard": "^16.0.1", "standard": "^16.0.1",
"start-server-and-test": "^1.11.3" "start-server-and-test": "^1.11.3"
}, },
@ -47,11 +50,10 @@
] ]
}, },
"standard": { "standard": {
"parser": "@babel/eslint-parser" "ignore": "dist"
}, },
"files": [ "files": [
"src", "dist",
"!*.spec.js",
"typings" "typings"
] ]
} }

23
rollup.config.js Normal file
View File

@ -0,0 +1,23 @@
module.exports = [{
// nearly indistinguishable from a copy and paste of the source
input: 'src/index.js',
output: {
file: 'dist/es6/base65536.js',
format: 'esm'
}
}, {
// same as above but Node.js-friendly
input: 'src/index.js',
output: {
file: 'dist/cjs/base65536.js',
format: 'cjs'
}
}, {
// this can be imported into the browser as a script
input: 'src/index.js',
output: {
file: 'dist/iife/base65536.js',
format: 'iife',
name: 'base65536'
}
}]

View File

@ -21,40 +21,44 @@
import SafeCodePoint from 'safe-code-point' import SafeCodePoint from 'safe-code-point'
// Well these ergonomics are dreadful huh // Well these ergonomics are dreadful huh
const safeCodePoint = await SafeCodePoint('8.0.0') export default () => SafeCodePoint('8.0.0').then(safeCodePoint => {
const safeRange = (min, max) => {
const safeRange = (min, max) => { for (let codePoint = min; codePoint < max; codePoint++) {
for (let codePoint = min; codePoint < max; codePoint++) { // Code points were chosen entirely from the "Letter, other" general
// Code points were chosen entirely from the "Letter, other" general // category, for reasons which I no longer recall. Unicode 8.0 was current
// category, for reasons which I no longer recall. Unicode 8.0 was current // at the time.
// at the time. if (
if ( safeCodePoint.generalCategory(codePoint) !== 'Lo' ||
safeCodePoint.generalCategory(codePoint) !== 'Lo' || !safeCodePoint(codePoint)
!safeCodePoint(codePoint) ) {
) { return false
return false }
} }
return true
} }
return true
}
const getAllSafeRanges = rangeSize => { const getAllSafeRanges = rangeSize => {
const allSafeRanges = [] const allSafeRanges = []
for (let codePoint = 0; codePoint < (1 << 16) + (1 << 20); codePoint += rangeSize) { for (let codePoint = 0; codePoint < (1 << 16) + (1 << 20); codePoint += rangeSize) {
if (safeRange(codePoint, codePoint + rangeSize)) { if (safeRange(codePoint, codePoint + rangeSize)) {
allSafeRanges.push(codePoint) allSafeRanges.push(codePoint)
}
} }
return allSafeRanges
} }
return allSafeRanges
}
const allSafeRanges = getAllSafeRanges(1 << 8) const allSafeRanges = getAllSafeRanges(1 << 8)
const paddingBlockStart = String.fromCodePoint(allSafeRanges.shift()) const paddingBlockStart = String.fromCodePoint(allSafeRanges.shift())
const blockStarts = allSafeRanges.slice(0, 1 << 8).map(x => String.fromCodePoint(x)).join('') const blockStarts = allSafeRanges.slice(0, 1 << 8).map(x => String.fromCodePoint(x)).join('')
export { safeCodePoint, paddingBlockStart, blockStarts } return {
safeCodePoint,
paddingBlockStart,
blockStarts
}
})
// There are now implementations of // There are now implementations of
// Base65536 in numerous programming languages beyond the original JavaScript, // Base65536 in numerous programming languages beyond the original JavaScript,

View File

@ -1,14 +1,22 @@
/* eslint-env jest */ /* eslint-env jest */
import { paddingBlockStart, blockStarts, safeCodePoint } from './gen' import gen from './gen'
describe('gen', () => { describe('gen', () => {
let generated
beforeAll(() =>
gen().then(g => {
generated = g
})
)
it('generates the correct padding block', () => { it('generates the correct padding block', () => {
expect(paddingBlockStart).toBe('ᔀ') expect(generated.paddingBlockStart).toBe('ᔀ')
}) })
it('generates the correct blocks', () => { it('generates the correct blocks', () => {
expect(blockStarts).toBe( expect(generated.blockStarts).toBe(
'㐀㔀㘀㜀㠀㤀㨀㬀㰀㴀㸀㼀䀀䄀䈀䌀' + '㐀㔀㘀㜀㠀㤀㨀㬀㰀㴀㸀㼀䀀䄀䈀䌀' +
'䐀䔀䘀䜀䠀䤀䨀䬀䰀一伀倀儀刀匀吀' + '䐀䔀䘀䜀䠀䤀䨀䬀䰀一伀倀儀刀匀吀' +
'唀嘀圀堀夀娀嬀尀崀帀开怀愀戀挀搀' + '唀嘀圀堀夀娀嬀尀崀帀开怀愀戀挀搀' +
@ -33,7 +41,7 @@ describe('gen', () => {
// 243 of the blocks are 'W' (wide), the other 13 + 1 are 'N' (neutral, // 243 of the blocks are 'W' (wide), the other 13 + 1 are 'N' (neutral,
// which in effect is narrow). This is significant when considering // which in effect is narrow). This is significant when considering
// rendering and wrapping. // rendering and wrapping.
const allBlockStarts = [...blockStarts].map(x => x.codePointAt(0)) const allBlockStarts = [...generated.blockStarts].map(x => x.codePointAt(0))
const neutralBlockStarts = [...'ᔀꔀ𐘀𒀀𒄀𒈀𓀀𓄀𓈀𓌀𔐀𔔀𖠀𖤀'].map(x => x.codePointAt(0)) const neutralBlockStarts = [...'ᔀꔀ𐘀𒀀𒄀𒈀𓀀𓄀𓈀𓌀𔐀𔔀𖠀𖤀'].map(x => x.codePointAt(0))
allBlockStarts.forEach(blockStart => { allBlockStarts.forEach(blockStart => {
for (let i = 0; i < 1 << 8; i++) { for (let i = 0; i < 1 << 8; i++) {
@ -43,7 +51,7 @@ describe('gen', () => {
neutralBlockStart <= codePoint && neutralBlockStart <= codePoint &&
codePoint < neutralBlockStart + (1 << 8) codePoint < neutralBlockStart + (1 << 8)
) )
expect(safeCodePoint.eastAsianWidth(codePoint)).toBe(isInNeutralBlock ? 'N' : 'W') expect(generated.safeCodePoint.eastAsianWidth(codePoint)).toBe(isInNeutralBlock ? 'N' : 'W')
} }
}) })
}) })

View File

@ -1,16 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>Base65536</title>
<script type="module">
import { encode, decode } from '../src/index.js'
const encoded = encode(new TextEncoder('utf-8').encode('what the heck is up'))
const decoded = new TextDecoder('utf-8').decode(decode(encoded))
document.querySelector('.root').appendChild(document.createTextNode(decoded))
</script>
</head>
<body>
<div class="root"></div>
</body>
</html>

View File

@ -15,8 +15,8 @@ const BITS_PER_BYTE = 8
// Compressed representation of inclusive-exclusive ranges of characters used in this encoding. // Compressed representation of inclusive-exclusive ranges of characters used in this encoding.
const pairStrings = [ const pairStrings = [
'㐀䳿一黿ꄀꏿꔀꗿ𐘀<EFBFBD>𒀀<EFBFBD>𓀀<EFBFBD>𔐀<EFBFBD>𖠀<EFBFBD>𠀀<EFBFBD>', '㐀䴀一鼀ꄀꐀꔀꘀ𐘀𐜀𒀀𒌀𓀀𓐀𔐀𔘀𖠀𖨀𠀀𨘀',
'ᔀ' 'ᔀ'
] ]
// Decompression // Decompression
@ -28,7 +28,7 @@ pairStrings.forEach((pairString, r) => {
let z2 = 0 let z2 = 0
pairString.match(/../gu).forEach(pair => { pairString.match(/../gu).forEach(pair => {
const [first, last] = [...pair].map(x => x.codePointAt(0)) const [first, last] = [...pair].map(x => x.codePointAt(0))
for (let codePoint = first; codePoint <= last; codePoint++) { for (let codePoint = first; codePoint < last; codePoint++) {
const chr = String.fromCodePoint(codePoint) const chr = String.fromCodePoint(codePoint)
// SPECIAL CASE: flip the bytes around, because Base65536 was constructed to take the bytes // SPECIAL CASE: flip the bytes around, because Base65536 was constructed to take the bytes

8
test-built/cjs.js Normal file
View File

@ -0,0 +1,8 @@
const { encode, decode } = require('..')
const before = [1, 2, 3]
const after = Array.from(decode(encode(Uint8Array.from(before))))
if (JSON.stringify(after) !== JSON.stringify(before)) {
throw Error()
}

10
test-built/es6.js Normal file
View File

@ -0,0 +1,10 @@
// I want to write '..' here, but I can't figure out a way to get Babel to
// follow `package.json`'s "module" property instead of its "main" property
import { encode, decode } from '../dist/es6/base65536'
const before = [1, 2, 3]
const after = Array.from(decode(encode(Uint8Array.from(before))))
if (JSON.stringify(after) !== JSON.stringify(before)) {
throw Error()
}

8
test-built/iife.js Normal file
View File

@ -0,0 +1,8 @@
/* global describe, it, cy */
describe('base65536 IIFE', () => {
it('works', () => {
cy.visit('http://localhost:3000/test-built/index.html')
cy.contains('what the heck is up')
})
})

17
test-built/index.html Normal file
View File

@ -0,0 +1,17 @@
<!DOCTYPE html>
<html>
<head>
<title></title>
<script crossorigin src="../dist/iife/base65536.js"></script>
<script>
window.addEventListener('load', () => {
const encoded = base65536.encode(new TextEncoder('utf-8').encode('what the heck is up'))
const decoded = new TextDecoder('utf-8').decode(base65536.decode(encoded))
document.querySelector('.root').appendChild(document.createTextNode(decoded))
})
</script>
</head>
<body>
<div class="root"></div>
</body>
</html>

View File

@ -1,5 +1,4 @@
import express from 'express' const express = require('express')
const app = express() const app = express()
app.use(express.static('.')) app.use(express.static('.'))
app.listen(3000, () => { app.listen(3000, () => {