From: Stefan Sperling Subject: Re: sha256 pack support To: Omar Polo Cc: gameoftrees@openbsd.org Date: Sat, 13 Jul 2024 16:41:14 +0200 On Sat, Jul 13, 2024 at 01:51:00PM +0200, Omar Polo wrote: > This needs the 'prepare got_object_parse_tree for sha256' diff (actually > not, but you'll need to sprinkle some more #include "got_lib_hash.h" > otherwise.) > > This changes the way we read the packidx trailer to accomodate for > sha256 digests. ok? ok, looks good > > commit 718bfbdf87bfc83281550a99af7be8d574d61ac6 (main) > from: Omar Polo > date: Sat Jul 13 11:48:49 2024 UTC > > sha256 packs support > > diff b94189a53000d148c342656c7467f12e4ff0a37a 718bfbdf87bfc83281550a99af7be8d574d61ac6 > commit - b94189a53000d148c342656c7467f12e4ff0a37a > commit + 718bfbdf87bfc83281550a99af7be8d574d61ac6 > blob - a061cd0784026046d9eed1bc09c78760626ce3e5 > blob + 247eb237105fc63a1460bc5b652b185f1530e548 > --- gotd/privsep_stub.c > +++ gotd/privsep_stub.c > @@ -33,6 +33,7 @@ > #include "got_path.h" > > #include "got_lib_delta.h" > +#include "got_lib_hash.h" > #include "got_lib_object.h" > #include "got_lib_object_cache.h" > #include "got_lib_pack.h" > blob - 878ae4aaae7f05abb0763bdfef2b229d62c7383e > blob + c0161ad25f562b9a152de2dc3d46259e2360c32b > --- lib/fetch.c > +++ lib/fetch.c > @@ -52,13 +52,13 @@ > #include "got_opentemp.h" > #include "got_fetch.h" > > +#include "got_lib_hash.h" > #include "got_lib_delta.h" > #include "got_lib_inflate.h" > #include "got_lib_object.h" > #include "got_lib_object_parse.h" > #include "got_lib_object_create.h" > #include "got_lib_pack.h" > -#include "got_lib_hash.h" > #include "got_lib_privsep.h" > #include "got_lib_object_cache.h" > #include "got_lib_repository.h" > blob - d29cab547a3e6b7153c70dab10764138ee97b990 > blob + 374e644c5f8bf340e2ac98aca1d0228ccc0f2a75 > --- lib/got_lib_pack.h > +++ lib/got_lib_pack.h > @@ -64,8 +64,8 @@ const struct got_error *got_pack_parse_object_type_and > /* See Documentation/technical/pack-format.txt in Git. */ > > struct got_packidx_trailer { > - u_int8_t packfile_sha1[SHA1_DIGEST_LENGTH]; > - u_int8_t packidx_sha1[SHA1_DIGEST_LENGTH]; > + u_int8_t packfile_hash[GOT_HASH_DIGEST_MAXLEN]; > + u_int8_t packidx_hash[GOT_HASH_DIGEST_MAXLEN]; > } __attribute__((__packed__)); > > /* Ignore pack index version 1 which is no longer written by Git. */ > @@ -101,7 +101,7 @@ struct got_packidx_v2_hdr { > /* Large offsets table is empty for pack files < 2 GB. */ > uint64_t *large_offsets; /* values are big endian */ > > - struct got_packidx_trailer *trailer; > + struct got_packidx_trailer trailer; > }; > > struct got_pack_offset_index { > blob - a18b2539b7434d2225f3d7b94c76e7dbad21999b > blob + 60307d5b70c26ee153a22138d08852f616acf52a > --- lib/object_open_io.c > +++ lib/object_open_io.c > @@ -34,13 +34,13 @@ > #include "got_path.h" > > #include "got_lib_delta.h" > +#include "got_lib_hash.h" > #include "got_lib_object.h" > #include "got_lib_object_cache.h" > #include "got_lib_object_parse.h" > #include "got_lib_pack.h" > #include "got_lib_repository.h" > #include "got_lib_inflate.h" > -#include "got_lib_hash.h" > > const struct got_error * > got_object_open_packed(struct got_object **obj, struct got_object_id *id, > blob - 20ad907906f81fc5442c688832a4e06b05beee55 > blob + 530976c9d02bf1c055238e2cad8109ee4b7dbe79 > --- lib/pack.c > +++ lib/pack.c > @@ -78,11 +78,12 @@ got_packidx_init_hdr(struct got_packidx *p, int verify > struct got_packidx_v2_hdr *h; > struct got_hash ctx; > uint8_t hash[GOT_HASH_DIGEST_MAXLEN]; > - size_t nobj, len_fanout, len_ids, offset, remain; > + size_t nobj, len_fanout, len_ids, offset, remain, idlen; > ssize_t n; > int i; > > got_hash_init(&ctx, p->algo); > + idlen = got_hash_digest_length(p->algo); > > h = &p->hdr; > offset = 0; > @@ -302,32 +303,33 @@ got_packidx_init_hdr(struct got_packidx *p, int verify > offset += p->nlargeobj * sizeof(*h->large_offsets); > > checksum: > - if (remain < sizeof(*h->trailer)) { > + if (remain < idlen * 2) { > err = got_error(GOT_ERR_BAD_PACKIDX); > goto done; > } > - if (p->map) > - h->trailer = > - (struct got_packidx_trailer *)((uint8_t*)(p->map + offset)); > - else { > - h->trailer = malloc(sizeof(*h->trailer)); > - if (h->trailer == NULL) { > - err = got_error_from_errno("malloc"); > + if (p->map) { > + memcpy(h->trailer.packfile_hash, p->map + offset, idlen); > + memcpy(h->trailer.packidx_hash, p->map + offset + idlen, idlen); > + } else { > + n = read(p->fd, h->trailer.packfile_hash, idlen); > + if (n < 0) > + err = got_error_from_errno("read"); > + else if (n != idlen) { > + err = got_error(GOT_ERR_BAD_PACKIDX); > goto done; > } > - n = read(p->fd, h->trailer, sizeof(*h->trailer)); > + n = read(p->fd, h->trailer.packidx_hash, idlen); > if (n < 0) > err = got_error_from_errno("read"); > - else if (n != sizeof(*h->trailer)) { > + else if (n != idlen) { > err = got_error(GOT_ERR_BAD_PACKIDX); > goto done; > } > } > if (verify) { > - got_hash_update(&ctx, h->trailer->packfile_sha1, > - got_hash_digest_length(p->algo)); > + got_hash_update(&ctx, h->trailer.packfile_hash, idlen); > got_hash_final(&ctx, hash); > - if (got_hash_cmp(ctx.algo, hash, h->trailer->packidx_sha1) != 0) > + if (got_hash_cmp(ctx.algo, hash, h->trailer.packidx_hash) != 0) > err = got_error(GOT_ERR_PACKIDX_CSUM); > } > done: > @@ -435,7 +437,6 @@ got_packidx_close(struct got_packidx *packidx) > free(packidx->hdr.crc32); > free(packidx->hdr.offsets); > free(packidx->hdr.large_offsets); > - free(packidx->hdr.trailer); > } > if (close(packidx->fd) == -1 && err == NULL) > err = got_error_from_errno("close"); > blob - 71dc7c652b8d8bc52bbbd8f75326fca9344bd271 > blob + 5aa56e154c58be00385c847ac667f05de6d70d6e > --- lib/pack_create_io.c > +++ lib/pack_create_io.c > @@ -40,6 +40,7 @@ > #include "got_path.h" > > #include "got_lib_delta.h" > +#include "got_lib_hash.h" > #include "got_lib_object.h" > #include "got_lib_object_cache.h" > #include "got_lib_object_idset.h" > blob - 99d9089dac46fe448e4697e91a93e3a1a19a634b > blob + 10b2b156100b2de5d6fa3f09483b5ac4658b3074 > --- lib/read_gitconfig.c > +++ lib/read_gitconfig.c > @@ -36,6 +36,7 @@ > > #include "got_lib_gitconfig.h" > #include "got_lib_delta.h" > +#include "got_lib_hash.h" > #include "got_lib_object.h" > #include "got_lib_object_cache.h" > #include "got_lib_privsep.h" > blob - b6be87a769c4d7440f941e66aabb66e589e19a77 > blob + d6d1733a9530e64b2f68b2f1ec8489706f582b2a > --- lib/send.c > +++ lib/send.c > @@ -57,12 +57,12 @@ > #include "got_commit_graph.h" > > #include "got_lib_delta.h" > +#include "got_lib_hash.h" > #include "got_lib_inflate.h" > #include "got_lib_object.h" > #include "got_lib_object_parse.h" > #include "got_lib_object_create.h" > #include "got_lib_pack.h" > -#include "got_lib_hash.h" > #include "got_lib_privsep.h" > #include "got_lib_object_cache.h" > #include "got_lib_repository.h" > >