For a nodejs project I need to determinate the hash of my folder to check the version. Actually, I made a script to test my code (without filesystem, directly of git api for my test). But it works half the time.
I used this API to get the hash : https://api.github.com/repos/zestedesavoir/zds-site/branches/dev
I made a Perl script version to check my code js code. It returns the same things. I think my error is in Object.values(json.tree).forEach(function (blob)
the pattern must be not good with text += blob.mode + " " + blob.path + "\0" + sha;
. I don't know why.
My js script :
(Live demo : https://repl.it/repls/FearfulWhiteShelfware)
const crypto = require("crypto"),
fs = require("fs"),
path = require("path"),
getURL = require("./ajax.js").getURL;
const apiJSON = [];
//https://api.github.com/
const hashs = [
"8d66139b3acf78fa50e16383693a161c33b5e048",
"4ef57de8e81c8415d6da2b267872e602b1f28cfe",
"13b54c0bab5e7f7a05398d6d92e65eee2b227136",
"218a8f506fcd3076fad059ec42d4656c635a8171"
];
let loaded = 0;
const USEAPI = false; /* becarful low limit on repl.it */
for (let i = 0; i < hashs.length; i++) {
if (!USEAPI) {
apiJSON[i] = JSON.parse(fs.readFileSync(`a${i+1}.json`));
console.log(`A${i+1}:`);
getTreeSHA(apiJSON[i], false);
if (i+1 === hashs.length) {
console.log("\n\nPerl ouput:");
for (let j = 0; j < hashs.length; j++)
getTreeSHA(apiJSON[j], true);
}
} else {
getURL("/repos/zestedesavoir/zds-site/git/trees/" + hashs[i], function(json) {
loaded++; apiJSON[i] = json;
if (loaded === hashs.length) {
for (let i = 0; i < hashs.length; i++) {
console.log(`A${i+1}:`); getTreeSHA(apiJSON[i], false);
}
console.log("\n\nPerl ouput:");
for (let i = 0; i < hashs.length; i++)
getTreeSHA(apiJSON[i], true);
}
});
}
}
function getTreeSHA(json, getPattern) {
/*json.tree.sort((a, b) => { ---> not good see A3 & A4
if (a.type !== b.type)
if (a.type === "tree")
return 1;
else if (b.type === "tree")
return -1;
return a.path.charCodeAt(0) - b.path.charCodeAt(0)
});*/
let text = "";
Object.values(json.tree).forEach(function (blob) {
const sha = Buffer.from(blob.sha, "hex").toString(!getPattern ? "binary" : "hex");
text += (+blob.mode) + " " + blob.path;
// ^ https://stackoverflow.com/a/54137728
text += (!getPattern) ? ("\0" + sha) : (" " + sha + "\n");
});
if (getPattern) return console.log(text.replace(/\0/g, ""));
console.log("Original " + json.sha);
const pattern = "tree " + text.length + "\0" + text;
console.log("Actual : " + sha1(pattern));
function sha1(data) {
return crypto.createHash("sha1").update(data, "binary").digest("hex");
}
}
Output :
A1:
Original 8d66139b3acf78fa50e16383693a161c33b5e048
Actual : 8d66139b3acf78fa50e16383693a161c33b5e048
A2:
Original 4ef57de8e81c8415d6da2b267872e602b1f28cfe
Actual : c5c701b8114582e3bb2e353aac157a7febfcd33b <-- not god
A3:
Original 13b54c0bab5e7f7a05398d6d92e65eee2b227136
Actual : 13b54c0bab5e7f7a05398d6d92e65eee2b227136
A4:
Original 218a8f506fcd3076fad059ec42d4656c635a8171
Actual : 218a8f506fcd3076fad059ec42d4656c635a8171
Wanted Output :
//...
A2:
Original 4ef57de8e81c8415d6da2b267872e602b1f28cfe
Actual : 4ef57de8e81c8415d6da2b267872e602b1f28cfe
//...
A2 :
{
"sha": "4ef57de8e81c8415d6da2b267872e602b1f28cfe",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/4ef57de8e81c8415d6da2b267872e602b1f28cfe",
"tree": [
{
"path": ".coveragerc",
"mode": "100644",
"type": "blob",
"sha": "449170d0faeb75182310345564fd1811c0b9fd73",
"size": 163,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/449170d0faeb75182310345564fd1811c0b9fd73"
},
{
"path": ".editorconfig",
"mode": "100644",
"type": "blob",
"sha": "75884936ea2d35b531af886acad747d4fd9b2a9e",
"size": 328,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/75884936ea2d35b531af886acad747d4fd9b2a9e"
},
{
"path": ".flake8",
"mode": "100644",
"type": "blob",
"sha": "69e872e30d30f5c7de3276d289d6aee81ccf4af7",
"size": 232,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/69e872e30d30f5c7de3276d289d6aee81ccf4af7"
},
{
"path": ".github",
"mode": "040000",
"type": "tree",
"sha": "56b49acad224fdb70fca11809f3e5a4d396cb01c",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/56b49acad224fdb70fca11809f3e5a4d396cb01c"
},
{
"path": ".gitignore",
"mode": "100644",
"type": "blob",
"sha": "4832b44b973574253cf1b59ba7a66cfc227cd699",
"size": 1439,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/4832b44b973574253cf1b59ba7a66cfc227cd699"
},
{
"path": ".jshintrc",
"mode": "100644",
"type": "blob",
"sha": "939efa02939437adece1e3a076d597b2557e36b5",
"size": 319,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/939efa02939437adece1e3a076d597b2557e36b5"
},
{
"path": ".travis.yml",
"mode": "100644",
"type": "blob",
"sha": "6b5e4f43790874e2cf9db23e964f72b99deeb0d1",
"size": 6040,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/6b5e4f43790874e2cf9db23e964f72b99deeb0d1"
},
{
"path": "AUTHORS",
"mode": "100644",
"type": "blob",
"sha": "0b92b7759ce2dd0a7cacf79b273368bb71ac5397",
"size": 197,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/0b92b7759ce2dd0a7cacf79b273368bb71ac5397"
},
{
"path": "CODE_OF_CONDUCT.md",
"mode": "100644",
"type": "blob",
"sha": "ae61c31efae6cea565e447467e4377da76125679",
"size": 2754,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/ae61c31efae6cea565e447467e4377da76125679"
},
{
"path": "CONTRIBUTING.md",
"mode": "100644",
"type": "blob",
"sha": "ac71ad378faf7fb7ae927b20d4d28a57c6085bf9",
"size": 155,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/ac71ad378faf7fb7ae927b20d4d28a57c6085bf9"
},
{
"path": "COPYING",
"mode": "100644",
"type": "blob",
"sha": "94a9ed024d3859793618152ea559a168bbcbb5e2",
"size": 35147,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/94a9ed024d3859793618152ea559a168bbcbb5e2"
},
{
"path": "Gulpfile.js",
"mode": "100644",
"type": "blob",
"sha": "5dd951ae61f0913605197fafa018f7db49549a68",
"size": 6137,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/5dd951ae61f0913605197fafa018f7db49549a68"
},
{
"path": "LICENSE",
"mode": "100644",
"type": "blob",
"sha": "8a171a155d85927b678068becd046194aea777a9",
"size": 717,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/8a171a155d85927b678068becd046194aea777a9"
},
{
"path": "Makefile",
"mode": "100644",
"type": "blob",
"sha": "cc722c2bc71dfbaa1b025c8c56245ed0fcd61739",
"size": 3829,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/cc722c2bc71dfbaa1b025c8c56245ed0fcd61739"
},
{
"path": "README.md",
"mode": "100644",
"type": "blob",
"sha": "a6a9013159a3766da62443c4be5e267435469fd9",
"size": 3280,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/a6a9013159a3766da62443c4be5e267435469fd9"
},
{
"path": "assets",
"mode": "040000",
"type": "tree",
"sha": "1846a32450eb2a7605acb55cab8206028cfb656f",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/1846a32450eb2a7605acb55cab8206028cfb656f"
},
{
"path": "doc",
"mode": "040000",
"type": "tree",
"sha": "f55b804a2b694db577b20c8e9851ad783fea8ee5",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/f55b804a2b694db577b20c8e9851ad783fea8ee5"
},
{
"path": "errors",
"mode": "040000",
"type": "tree",
"sha": "b37a18162be2bdae7382fc194f1bf2d0ab89bba3",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/b37a18162be2bdae7382fc194f1bf2d0ab89bba3"
},
{
"path": "export-assets",
"mode": "040000",
"type": "tree",
"sha": "3a8b85efa969c389ac3c5e7e6ad62206dbddcaca",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/3a8b85efa969c389ac3c5e7e6ad62206dbddcaca"
},
{
"path": "fixtures",
"mode": "040000",
"type": "tree",
"sha": "89cacb4de6feb81a962b9a992b9434cb44d3b0aa",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/89cacb4de6feb81a962b9a992b9434cb44d3b0aa"
},
{
"path": "geodata",
"mode": "040000",
"type": "tree",
"sha": "635d29035ae7528231edb9b74eb09887c22dda2a",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/635d29035ae7528231edb9b74eb09887c22dda2a"
},
{
"path": "manage.py",
"mode": "100755",
"type": "blob",
"sha": "458f6e2df8b431b9fa819c89e82cebf2e0a91260",
"size": 1536,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/458f6e2df8b431b9fa819c89e82cebf2e0a91260"
},
{
"path": "package.json",
"mode": "100644",
"type": "blob",
"sha": "02d231aa0c0fa299581be07bcece0393dc9a9e47",
"size": 1402,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/02d231aa0c0fa299581be07bcece0393dc9a9e47"
},
{
"path": "quotes.txt",
"mode": "100644",
"type": "blob",
"sha": "e8e84a048d70bc57c1f725fc12f2101a40c5dcbb",
"size": 1552,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/e8e84a048d70bc57c1f725fc12f2101a40c5dcbb"
},
{
"path": "requirements-dev.txt",
"mode": "100644",
"type": "blob",
"sha": "7297a894036fcf70a7209062bb51f45db1b71d39",
"size": 227,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/7297a894036fcf70a7209062bb51f45db1b71d39"
},
{
"path": "requirements-prod.txt",
"mode": "100644",
"type": "blob",
"sha": "2f957115bcf3794fdecf3c4848f21ae8f428c31b",
"size": 83,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/2f957115bcf3794fdecf3c4848f21ae8f428c31b"
},
{
"path": "requirements.txt",
"mode": "100644",
"type": "blob",
"sha": "805fefa566ef0d8f6a7c7e58d01fa4684078cf50",
"size": 998,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/805fefa566ef0d8f6a7c7e58d01fa4684078cf50"
},
{
"path": "robots.txt",
"mode": "100644",
"type": "blob",
"sha": "8ca70253a4bb677cb797a7b409df4c4a9c0baa67",
"size": 948,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/8ca70253a4bb677cb797a7b409df4c4a9c0baa67"
},
{
"path": "scripts",
"mode": "040000",
"type": "tree",
"sha": "f6a251faaaa14ba4fcf702cd0556675e70cc80f3",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/f6a251faaaa14ba4fcf702cd0556675e70cc80f3"
},
{
"path": "suggestions.txt",
"mode": "100644",
"type": "blob",
"sha": "5e5d11a62a00d3f1aea8f3825c8ec89860d31ad0",
"size": 285,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/5e5d11a62a00d3f1aea8f3825c8ec89860d31ad0"
},
{
"path": "templates",
"mode": "040000",
"type": "tree",
"sha": "5b6dde8b8b616ba078305584e23e55ad0c5b2299",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/5b6dde8b8b616ba078305584e23e55ad0c5b2299"
},
{
"path": "update.md",
"mode": "100644",
"type": "blob",
"sha": "734cb67218ac7ad952ffe2f816e4820427efe809",
"size": 45743,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/734cb67218ac7ad952ffe2f816e4820427efe809"
},
{
"path": "yarn.lock",
"mode": "100644",
"type": "blob",
"sha": "9fed208fbed286860cb606c9904eb3bab2b3d960",
"size": 193867,
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/blobs/9fed208fbed286860cb606c9904eb3bab2b3d960"
},
{
"path": "zds",
"mode": "040000",
"type": "tree",
"sha": "45b76aa70ad46e116c491a55def4b396b4ecba89",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/45b76aa70ad46e116c491a55def4b396b4ecba89"
},
{
"path": "zmd",
"mode": "040000",
"type": "tree",
"sha": "89289051d5d1e37ecc12629737d4fc01dd0df06e",
"url": "https://api.github.com/repos/zestedesavoir/zds-site/git/trees/89289051d5d1e37ecc12629737d4fc01dd0df06e"
}
],
"truncated": false
}
I made a Perl script version to check my code and understand this issue.
Perl :
(Live demo : https://repl.it/repls/VainPrizeDebugmonitor)
Files :
output_a1
100644 arborescence-back.rst 05392dacd107b9e2bb931c85632e115ae69c22cb
100644 featured.rst 20084355452644d3a171b54e9331485e73a897ea
100644 forum.rst 82efe44d491fbc69fb99b0fc0829ad349a11aae7
100644 gallery.rst e075f6d1fe182e595b950cc50d1c5701c6c48bb1
100644 member.rst 157a97545f397f02293a95034989f293cda00ee8
100644 pages.rst 30d85eb8babc8608a87272eb02a73685a71623c3
100644 private-message.rst 6e4872283841ddb0edf03f7535003b4cb5e2f3ce
100644 searchv2.rst a31835f3f39b77408b75548c215d06dcd776d3c2
100644 tutorialv2.rst e646fef1203c7c9b8137c6420d990fd40c1255ae
100644 utils.rst 846765fc32bafc05bb58e6b70883acf5de8ae97b
output_a2
100644 .coveragerc 449170d0faeb75182310345564fd1811c0b9fd73
100644 .editorconfig 75884936ea2d35b531af886acad747d4fd9b2a9e
100644 .flake8 69e872e30d30f5c7de3276d289d6aee81ccf4af7
40000 .github 56b49acad224fdb70fca11809f3e5a4d396cb01c
100644 .gitignore 4832b44b973574253cf1b59ba7a66cfc227cd699
100644 .jshintrc 939efa02939437adece1e3a076d597b2557e36b5
100644 .travis.yml 6b5e4f43790874e2cf9db23e964f72b99deeb0d1
100644 AUTHORS 0b92b7759ce2dd0a7cacf79b273368bb71ac5397
100644 CODE_OF_CONDUCT.md ae61c31efae6cea565e447467e4377da76125679
100644 CONTRIBUTING.md ac71ad378faf7fb7ae927b20d4d28a57c6085bf9
100644 COPYING 94a9ed024d3859793618152ea559a168bbcbb5e2
100644 Gulpfile.js 5dd951ae61f0913605197fafa018f7db49549a68
100644 LICENSE 8a171a155d85927b678068becd046194aea777a9
100644 Makefile cc722c2bc71dfbaa1b025c8c56245ed0fcd61739
100644 README.md a6a9013159a3766da62443c4be5e267435469fd9
40000 assets 1846a32450eb2a7605acb55cab8206028cfb656f
40000 doc f55b804a2b694db577b20c8e9851ad783fea8ee5
40000 errors b37a18162be2bdae7382fc194f1bf2d0ab89bba3
40000 export-assets 3a8b85efa969c389ac3c5e7e6ad62206dbddcaca
40000 fixtures 89cacb4de6feb81a962b9a992b9434cb44d3b0aa
40000 geodata 635d29035ae7528231edb9b74eb09887c22dda2a
100755 manage.py 458f6e2df8b431b9fa819c89e82cebf2e0a91260
100644 package.json 02d231aa0c0fa299581be07bcece0393dc9a9e47
100644 quotes.txt e8e84a048d70bc57c1f725fc12f2101a40c5dcbb
100644 requirements-dev.txt 7297a894036fcf70a7209062bb51f45db1b71d39
100644 requirements-prod.txt 2f957115bcf3794fdecf3c4848f21ae8f428c31b
100644 requirements.txt 805fefa566ef0d8f6a7c7e58d01fa4684078cf50
100644 robots.txt 8ca70253a4bb677cb797a7b409df4c4a9c0baa67
40000 scripts f6a251faaaa14ba4fcf702cd0556675e70cc80f3
100644 suggestions.txt 5e5d11a62a00d3f1aea8f3825c8ec89860d31ad0
40000 templates 5b6dde8b8b616ba078305584e23e55ad0c5b2299
100644 update.md 734cb67218ac7ad952ffe2f816e4820427efe809
100644 yarn.lock 9fed208fbed286860cb606c9904eb3bab2b3d960
40000 zds 45b76aa70ad46e116c491a55def4b396b4ecba89
40000 zmd 89289051d5d1e37ecc12629737d4fc01dd0df06e
output_a3
100644 Makefile fd4542fcb89018c3f97901b26992577590db1fe1
100644 make.bat f17fd5b680fc6dafdba3d1adda49389de4ae0b25
40000 source 7425440b50da313c10be22342f8a0f575ca64196
output_a4
40000 includes 52fe1c1c43130c011e78fc7d488ee5cd2d39fc61
100644 opensearch.xml be2e32c0f7c32a22da4c428438ae6f79965ea4ca
100644 search.html 5618f244fee4945eb799022f7e109ec8cbb2c696
Perl script :
XX="$(perl -sane '$F[2] =~ s/(..)/\\x$1/g ; print $F[0]." ".$F[1]."\\"."x00".$F[2]' output_a1)"
SIZE=$(echo -en "$XX" | wc -c)
echo "A1:"
echo "original: 8d66139b3acf78fa50e16383693a161c33b5e048"
echo "output :" $(echo -en "tree $SIZE\x00$XX" | sha1sum)
# ...
Output :
A1:
original: 8d66139b3acf78fa50e16383693a161c33b5e048
output : 8d66139b3acf78fa50e16383693a161c33b5e048
A2:
original: 4ef57de8e81c8415d6da2b267872e602b1f28cfe
output : c5c701b8114582e3bb2e353aac157a7febfcd33b
A3:
original: 13b54c0bab5e7f7a05398d6d92e65eee2b227136
output : 13b54c0bab5e7f7a05398d6d92e65eee2b227136
A4:
original: 218a8f506fcd3076fad059ec42d4656c635a8171
output : 218a8f506fcd3076fad059ec42d4656c635a8171
Then we see that my JS and Perl Script return the same thing. That mean, that my pattern was malformed, I don't know why.
Your code contains a problem, however fixing it doesn't remove the discrepancy in the A2 case.
The problem with your code
The official git algorithm for computing hashes on trees drops leading zeros from the mode field. In your examples, that field contains values
100644
and040000
, and the latter is recorded by git as40000
.Proof:
But adding removal of leading zeros1 to your perl script still doesn't fix the A2 case (although the computed hash changes, it is still different from the expected one):
The problem (?) with the GitHub API
The explanation is that the A2 hash 4ef57de8e81c8415d6da2b267872e602b1f28cfe points to a commit object rather than a tree. That commit object in turn refers to the tree with hash c5c701b8114582e3bb2e353aac157a7febfcd33b, which is exactly the value computed by the fixed code:
Had you used for the A2 query the tree rather than the commit hash (try this) you wouldn't have had any problem in the first place.
An arguable issue with the GitHub API is that it silently resolves a commit hash to the underlying tree instead of returning an error or including in the response an indication of what happened (for example, by setting the
sha
field to the hash of the tree rather than the query value).1 The quick&dirty fix won't work correctly in one case, when the mode field consists of only zeros. In that case the mode field will be completely erased instead of being replaced by a single zero. However that case cannot occur in practice, since an object with such a mode value would be simply inaccessible to git.