lkn
/
pcloudcc-lneely
mirror of https://github.com/lneely/pcloudcc-lneely.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pcrc32c namespace

This commit is contained in:
Levi Neely 2025-03-01 22:09:58 +01:00
parent 615c100967
commit c7facfebd4
5 changed files with 41 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pclsync/pcrc32c.c
View File

@ -436,7 +436,7 @@ static int crc_hashw = 0;
PSYNC_NOINLINE static uint32_t psync_crc32c_sw(uint32_t crc, const void *ptr, PSYNC_NOINLINE static uint32_t psync_crc32c_sw(uint32_t crc, const void *ptr,
size_t len) { size_t len) {
#else #else
uint32_t psync_crc32c(uint32_t crc, const void *ptr, size_t len) { uint32_t pcrc32c_compute(uint32_t crc, const void *ptr, size_t len) {
#endif #endif
const char *data; const char *data;
data = (const char *)ptr; data = (const char *)ptr;
@ -779,7 +779,7 @@ PSYNC_NOINLINE static uint32_t psync_crc32c_init(uint32_t crc, const void *ptr,
return psync_crc32c_sw(crc, ptr, len); return psync_crc32c_sw(crc, ptr, len);
} }
uint32_t psync_crc32c(uint32_t crc, const void *ptr, size_t len) { uint32_t pcrc32c_compute(uint32_t crc, const void *ptr, size_t len) {
#if defined(CRC32_GNUC) && defined(__SSE4_2__) #if defined(CRC32_GNUC) && defined(__SSE4_2__)
return psync_crc32c_hw(crc, ptr, len); return psync_crc32c_hw(crc, ptr, len);
#else #else
@ -800,7 +800,7 @@ uint32_t psync_crc32c(uint32_t crc, const void *ptr, size_t len) {
* https://code.google.com/p/cityhash/ * https://code.google.com/p/cityhash/
*/ */
void psync_fast_hash256_init(psync_fast_hash256_ctx *ctx) { void pcrc32c_fast_hash256_init(psync_fast_hash256_ctx *ctx) {
#ifdef CRC32_HW #ifdef CRC32_HW
if (unlikely(!crc_hashw)) if (unlikely(!crc_hashw))
crc_hashw = psync_has_hw_crc() + 1; crc_hashw = psync_has_hw_crc() + 1;
@ -810,10 +810,10 @@ void psync_fast_hash256_init(psync_fast_hash256_ctx *ctx) {
ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1] = 0; ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1] = 0;
} }
void psync_fast_hash256_init_seed(psync_fast_hash256_ctx *ctx, const void *seed, void pcrc32c_fast_hash256_seed(psync_fast_hash256_ctx *ctx, const void *seed,
size_t seedlen) { size_t seedlen) {
size_t i; size_t i;
psync_fast_hash256_init(ctx); pcrc32c_fast_hash256_init(ctx);
if (seedlen > sizeof(ctx->state)) if (seedlen > sizeof(ctx->state))
seedlen = sizeof(ctx->state); seedlen = sizeof(ctx->state);
for (i = 0; i < seedlen; i++) for (i = 0; i < seedlen; i++)
@ -950,7 +950,7 @@ static void psync_fast_hash256_update_long_sw(psync_fast_hash256_ctx *ctx,
static void psync_fast_hash256_update_short_sw(psync_fast_hash256_ctx *ctx, static void psync_fast_hash256_update_short_sw(psync_fast_hash256_ctx *ctx,
const char *cdata, size_t len); const char *cdata, size_t len);
void psync_fast_hash256_update(psync_fast_hash256_ctx *ctx, const void *data, void pcrc32c_fast_hash256_update(psync_fast_hash256_ctx *ctx, const void *data,
size_t len) { size_t len) {
ctx->length += len; ctx->length += len;
if (len + ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1] < if (len + ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1] <
@ -1023,7 +1023,7 @@ psync_fast_hash256_update_long_sw(psync_fast_hash256_ctx *ctx,
ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1] = len; ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1] = len;
} }
void psync_fast_hash256_final(void *hash, psync_fast_hash256_ctx *ctx) { void pcrc32c_fast_hash256_final(void *hash, psync_fast_hash256_ctx *ctx) {
uint64_t a, b, c, d, x, y, len; uint64_t a, b, c, d, x, y, len;
len = ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1]; len = ctx->buff[PSYNC_FAST_HASH256_BLOCK_LEN - 1];
memcpy(ctx->buff + len, &crc32c_table[1][2], memcpy(ctx->buff + len, &crc32c_table[1][2],

13
pclsync/pcrc32c.h
View File

@ -51,7 +51,7 @@ typedef struct {
}; };
} psync_fast_hash256_ctx; } psync_fast_hash256_ctx;
uint32_t psync_crc32c(uint32_t crc, const void *ptr, size_t len); uint32_t pcrc32c_compute(uint32_t crc, const void *ptr, size_t len);
/* psync_fast_hash256 is supposed to be fast, non-cryptographic strength, /* psync_fast_hash256 is supposed to be fast, non-cryptographic strength,
* non-collision resistant hash function with large output. Think of it as CRC32 * non-collision resistant hash function with large output. Think of it as CRC32
@ -59,12 +59,9 @@ uint32_t psync_crc32c(uint32_t crc, const void *ptr, size_t len);
* chances of CRC32 _random_ collision. It is intended for large inputs, * chances of CRC32 _random_ collision. It is intended for large inputs,
* finalization is relatively expensive. * finalization is relatively expensive.
*/ */
void pcrc32c_fast_hash256_init(psync_fast_hash256_ctx *ctx);
void psync_fast_hash256_init(psync_fast_hash256_ctx *ctx); void pcrc32c_fast_hash256_seed(psync_fast_hash256_ctx *ctx, const void *seed, size_t seedlen);
void psync_fast_hash256_init_seed(psync_fast_hash256_ctx *ctx, const void *seed, void pcrc32c_fast_hash256_update(psync_fast_hash256_ctx *ctx, const void *data, size_t len);
size_t seedlen); void pcrc32c_fast_hash256_final(void *hash, psync_fast_hash256_ctx *ctx);
void psync_fast_hash256_update(psync_fast_hash256_ctx *ctx, const void *data,
size_t len);
void psync_fast_hash256_final(void *hash, psync_fast_hash256_ctx *ctx);
#endif #endif

6
pclsync/pdiff.c
View File

@ -2848,10 +2848,10 @@ static void psync_diff_adapter_hash(void *out) {
psync_fast_hash256_ctx ctx; psync_fast_hash256_ctx ctx;
psock_ifaces_t *list; psock_ifaces_t *list;
list = psock_list_adapters(); list = psock_list_adapters();
psync_fast_hash256_init(&ctx); pcrc32c_fast_hash256_init(&ctx);
psync_fast_hash256_update(&ctx, list->interfaces, pcrc32c_fast_hash256_update(&ctx, list->interfaces,
list->interfacecnt * sizeof(psock_iface_t)); list->interfacecnt * sizeof(psock_iface_t));
psync_fast_hash256_final(out, &ctx); pcrc32c_fast_hash256_final(out, &ctx);
psync_free(list); psync_free(list);
} }

28
pclsync/pfscrypto.c
View File

@ -221,7 +221,7 @@ int pfscrypto_init_log(psync_openfile_t *of) {
return -EIO; return -EIO;
} }
of->logoffset = PSYNC_CRYPTO_SECTOR_SIZE; of->logoffset = PSYNC_CRYPTO_SECTOR_SIZE;
psync_fast_hash256_init(&of->loghashctx); pcrc32c_fast_hash256_init(&of->loghashctx);
return 0; return 0;
} }
@ -624,7 +624,7 @@ static int psync_fs_crypto_switch_sectors(psync_openfile_t *of,
(unsigned)sizeof(hdr), (int)wrt); (unsigned)sizeof(hdr), (int)wrt);
return -EIO; return -EIO;
} }
psync_fast_hash256_update(&of->loghashctx, &hdr, sizeof(hdr)); pcrc32c_fast_hash256_update(&of->loghashctx, &hdr, sizeof(hdr));
wrt = pfile_pwrite(of->logfile, &autharr[level], sz, wrt = pfile_pwrite(of->logfile, &autharr[level], sz,
of->logoffset + sizeof(hdr)); of->logoffset + sizeof(hdr));
if (unlikely(wrt != sz)) { if (unlikely(wrt != sz)) {
@ -632,7 +632,7 @@ static int psync_fs_crypto_switch_sectors(psync_openfile_t *of,
(int)wrt); (int)wrt);
return -EIO; return -EIO;
} }
psync_fast_hash256_update(&of->loghashctx, &autharr[level], sz); pcrc32c_fast_hash256_update(&of->loghashctx, &autharr[level], sz);
if (!of->newfile) if (!of->newfile)
psync_interval_tree_add(&of->writeintervals, hdr.offset, psync_interval_tree_add(&of->writeintervals, hdr.offset,
hdr.offset + sz); hdr.offset + sz);
@ -712,7 +712,7 @@ psync_fs_crypto_write_master_auth(psync_openfile_t *of,
(unsigned)sizeof(data), (int)wrt); (unsigned)sizeof(data), (int)wrt);
return -EIO; return -EIO;
} }
psync_fast_hash256_update(&of->loghashctx, &data, sizeof(data)); pcrc32c_fast_hash256_update(&of->loghashctx, &data, sizeof(data));
if (!of->newfile) if (!of->newfile)
psync_interval_tree_add(&of->writeintervals, offsets->masterauthoff, psync_interval_tree_add(&of->writeintervals, offsets->masterauthoff,
offsets->masterauthoff + PSYNC_CRYPTO_AUTH_SIZE); offsets->masterauthoff + PSYNC_CRYPTO_AUTH_SIZE);
@ -770,7 +770,7 @@ psync_fs_crypto_check_log_hash(int lfd,
debug(D_WARNING, "invalid hashid in log file %u", (unsigned)mr->hashid); debug(D_WARNING, "invalid hashid in log file %u", (unsigned)mr->hashid);
return -1; return -1;
} }
psync_fast_hash256_init(&ctx); pcrc32c_fast_hash256_init(&ctx);
off = PSYNC_CRYPTO_SECTOR_SIZE; off = PSYNC_CRYPTO_SECTOR_SIZE;
while ((rd = pfile_pread(lfd, buff, PSYNC_CRYPTO_SECTOR_SIZE, off)) != while ((rd = pfile_pread(lfd, buff, PSYNC_CRYPTO_SECTOR_SIZE, off)) !=
0) { 0) {
@ -779,10 +779,10 @@ psync_fs_crypto_check_log_hash(int lfd,
(int)errno); (int)errno);
return -1; return -1;
} }
psync_fast_hash256_update(&ctx, buff, rd); pcrc32c_fast_hash256_update(&ctx, buff, rd);
off += rd; off += rd;
} }
psync_fast_hash256_final(chash, &ctx); pcrc32c_fast_hash256_final(chash, &ctx);
if (memcmp(chash, mr->hash, PSYNC_FAST_HASH256_LEN)) { if (memcmp(chash, mr->hash, PSYNC_FAST_HASH256_LEN)) {
debug(D_WARNING, "calculated hash does not match"); debug(D_WARNING, "calculated hash does not match");
#if IS_DEBUG #if IS_DEBUG
@ -827,12 +827,12 @@ static int psync_fs_crypto_process_log(int lfd, int dfd,
return -1; return -1;
} }
if (unlikely(mr->crc != if (unlikely(mr->crc !=
psync_crc32c(PSYNC_CRC_INITIAL, mr, pcrc32c_compute(PSYNC_CRC_INITIAL, mr,
offsetof(psync_crypto_master_record, crc)))) { offsetof(psync_crypto_master_record, crc)))) {
debug(D_WARNING, debug(D_WARNING,
"got log file with bad master record CRC, expected %u got %u", "got log file with bad master record CRC, expected %u got %u",
(unsigned)mr->crc, (unsigned)mr->crc,
(unsigned)psync_crc32c(PSYNC_CRC_INITIAL, mr, (unsigned)pcrc32c_compute(PSYNC_CRC_INITIAL, mr,
offsetof(psync_crypto_master_record, crc))); offsetof(psync_crypto_master_record, crc)));
return -1; return -1;
} }
@ -953,8 +953,8 @@ static int psync_fs_crypto_mark_log_finalized(psync_openfile_t *of,
mr.hashid = PSYNC_LOG_HASHID_FH256; mr.hashid = PSYNC_LOG_HASHID_FH256;
mr.logsize = lsize; mr.logsize = lsize;
mr.filesize = filesize; mr.filesize = filesize;
psync_fast_hash256_final(&mr.hash, &of->loghashctx); pcrc32c_fast_hash256_final(&mr.hash, &of->loghashctx);
mr.crc = psync_crc32c(PSYNC_CRC_INITIAL, &mr, mr.crc = pcrc32c_compute(PSYNC_CRC_INITIAL, &mr,
offsetof(psync_crypto_master_record, crc)); offsetof(psync_crypto_master_record, crc));
wrt = pfile_pwrite(of->logfile, &mr, sizeof(mr), 0); wrt = pfile_pwrite(of->logfile, &mr, sizeof(mr), 0);
if (unlikely_log(wrt != sizeof(mr))) if (unlikely_log(wrt != sizeof(mr)))
@ -1020,7 +1020,7 @@ static int psync_fs_write_interval_tree_to_log(psync_openfile_t *of) {
(unsigned)sizeof(logs), (int)wrt); (unsigned)sizeof(logs), (int)wrt);
return -EIO; return -EIO;
} }
psync_fast_hash256_update(&of->loghashctx, logs, sizeof(logs)); pcrc32c_fast_hash256_update(&of->loghashctx, logs, sizeof(logs));
of->logoffset += sizeof(logs); of->logoffset += sizeof(logs);
last = 0; last = 0;
} }
@ -1041,7 +1041,7 @@ static int psync_fs_write_interval_tree_to_log(psync_openfile_t *of) {
(unsigned)(sizeof(psync_crypto_log_header) * last), (int)wrt); (unsigned)(sizeof(psync_crypto_log_header) * last), (int)wrt);
return -EIO; return -EIO;
} }
psync_fast_hash256_update(&of->loghashctx, logs, last); pcrc32c_fast_hash256_update(&of->loghashctx, logs, last);
of->logoffset += last; of->logoffset += last;
} }
return 0; return 0;
@ -1200,7 +1200,7 @@ psync_fs_crypto_write_newfile_full_sector(psync_openfile_t *of, const char *buf,
psync_fs_crypto_reset_log_to_off(of, of->logoffset); psync_fs_crypto_reset_log_to_off(of, of->logoffset);
return -EIO; return -EIO;
} }
psync_fast_hash256_update(&of->loghashctx, &rec, len); pcrc32c_fast_hash256_update(&of->loghashctx, &rec, len);
psync_fs_crypto_set_sector_log_offset(of, sectorid, of->logoffset, auth); psync_fs_crypto_set_sector_log_offset(of, sectorid, of->logoffset, auth);
of->logoffset += len; of->logoffset += len;
if (!of->newfile) if (!of->newfile)

24
pclsync/ppagecache.c
View File

@ -381,7 +381,7 @@ static int psync_pagecache_read_range_from_api(psync_request_t *request,
page->lastuse = ptimer_time(); page->lastuse = ptimer_time();
page->size = rb; page->size = rb;
page->usecnt = 0; page->usecnt = 0;
page->crc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, rb); page->crc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, rb);
page->type = PAGE_TYPE_READ; page->type = PAGE_TYPE_READ;
h = waiterhash_by_hash_and_pageid(page->hash, page->pageid); h = waiterhash_by_hash_and_pageid(page->hash, page->pageid);
lock_wait(page->hash); lock_wait(page->hash);
@ -848,7 +848,7 @@ static long check_page_in_memory_by_hash(uint64_t hash, uint64_t pageid,
page->usecnt++; page->usecnt++;
page->lastuse = tm; page->lastuse = tm;
} }
crc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, page->size); crc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, page->size);
if (unlikely(crc != page->crc)) { if (unlikely(crc != page->crc)) {
debug(D_WARNING, debug(D_WARNING,
"memory page CRC does not match %u!=%u, this is most likely memory " "memory page CRC does not match %u!=%u, this is most likely memory "
@ -1677,7 +1677,7 @@ static long check_page_in_database_by_hash(uint64_t hash,
ret = -1; ret = -1;
} else { } else {
if (unlikely(size == dsize && off == 0 && if (unlikely(size == dsize && off == 0 &&
psync_crc32c(PSYNC_CRC_INITIAL, buff, size) != crc)) { pcrc32c_compute(PSYNC_CRC_INITIAL, buff, size) != crc)) {
debug(D_WARNING, debug(D_WARNING,
"got bad CRC when reading data from cache at offset %lu", "got bad CRC when reading data from cache at offset %lu",
(unsigned long)(pagecacheid * PSYNC_FS_PAGE_SIZE + off)); (unsigned long)(pagecacheid * PSYNC_FS_PAGE_SIZE + off));
@ -1736,7 +1736,7 @@ static void check_pages_in_database_by_hash(uint64_t hash,
continue; continue;
} }
for (j = 0; j < cnt; j++) for (j = 0; j < cnt; j++)
if (psync_crc32c(PSYNC_CRC_INITIAL, if (pcrc32c_compute(PSYNC_CRC_INITIAL,
buff + (cpid - first_page_id + j) * PSYNC_FS_PAGE_SIZE, buff + (cpid - first_page_id + j) * PSYNC_FS_PAGE_SIZE,
PSYNC_FS_PAGE_SIZE) == PSYNC_FS_PAGE_SIZE) ==
psync_get_result_cell(fres, i + j, 3)) psync_get_result_cell(fres, i + j, 3))
@ -1799,7 +1799,7 @@ static long check_page_in_database_by_hash_and_cache(uint64_t hash,
psync_pagecache_return_free_page(page); psync_pagecache_return_free_page(page);
ret = -1; ret = -1;
} else { } else {
ccrc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, dsize); ccrc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, dsize);
if (unlikely(ccrc != crc)) { if (unlikely(ccrc != crc)) {
debug(D_WARNING, debug(D_WARNING,
"got bad CRC when reading data from cache at offset %lu, size " "got bad CRC when reading data from cache at offset %lu, size "
@ -1996,7 +1996,7 @@ static int psync_pagecache_read_range_from_sock(psync_request_t *request,
page->lastuse = ptimer_time(); page->lastuse = ptimer_time();
page->size = rb; page->size = rb;
page->usecnt = 0; page->usecnt = 0;
page->crc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, rb); page->crc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, rb);
page->type = PAGE_TYPE_READ; page->type = PAGE_TYPE_READ;
h = waiterhash_by_hash_and_pageid(page->hash, page->pageid); h = waiterhash_by_hash_and_pageid(page->hash, page->pageid);
lock_wait(page->hash); lock_wait(page->hash);
@ -3235,7 +3235,7 @@ static void psync_pagecache_new_upload_to_cache(uint64_t taskid, uint64_t hash,
page->lastuse = tm; page->lastuse = tm;
page->size = rd; page->size = rd;
page->usecnt = 1; page->usecnt = 1;
page->crc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, rd); page->crc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, rd);
page->type = PAGE_TYPE_READ; page->type = PAGE_TYPE_READ;
psync_pagecache_add_page_if_not_exists(page, hash, pageid); psync_pagecache_add_page_if_not_exists(page, hash, pageid);
if (rd < PSYNC_FS_PAGE_SIZE) if (rd < PSYNC_FS_PAGE_SIZE)
@ -3354,7 +3354,7 @@ static void psync_pagecache_modify_to_cache(uint64_t taskid, uint64_t hash,
page->lastuse = tm; page->lastuse = tm;
page->size = pdb; page->size = pdb;
page->usecnt = 1; page->usecnt = 1;
page->crc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, pdb); page->crc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, pdb);
page->type = PAGE_TYPE_READ; page->type = PAGE_TYPE_READ;
psync_pagecache_add_page_if_not_exists(page, hash, pageid); psync_pagecache_add_page_if_not_exists(page, hash, pageid);
break; break;
@ -3386,7 +3386,7 @@ static void psync_pagecache_modify_to_cache(uint64_t taskid, uint64_t hash,
page->lastuse = tm; page->lastuse = tm;
page->size = rd; page->size = rd;
page->usecnt = 1; page->usecnt = 1;
page->crc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, rd); page->crc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, rd);
page->type = PAGE_TYPE_READ; page->type = PAGE_TYPE_READ;
// debug(D_NOTICE, "new page %lu crc %lu size %lu", (unsigned // debug(D_NOTICE, "new page %lu crc %lu size %lu", (unsigned
// long)pageid, (unsigned long)page->crc, (unsigned long)rd); // long)pageid, (unsigned long)page->crc, (unsigned long)rd);
@ -3461,7 +3461,7 @@ static void psync_pagecache_modify_to_cache(uint64_t taskid, uint64_t hash,
page->lastuse = tm; page->lastuse = tm;
page->size = pdb; page->size = pdb;
page->usecnt = 1; page->usecnt = 1;
page->crc = psync_crc32c(PSYNC_CRC_INITIAL, page->page, pdb); page->crc = pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, pdb);
page->type = PAGE_TYPE_READ; page->type = PAGE_TYPE_READ;
// debug(D_NOTICE, "combined page %lu crc %lu size %lu", (unsigned // debug(D_NOTICE, "combined page %lu crc %lu size %lu", (unsigned
// long)pageid, (unsigned long)page->crc, (unsigned long)pdb); // long)pageid, (unsigned long)page->crc, (unsigned long)pdb);
@ -3723,13 +3723,13 @@ static int psync_pagecache_free_page_from_read_cache() {
page->usecnt = row[4]; page->usecnt = row[4];
page->crc = row[6]; page->crc = row[6];
psync_sql_free_result(res); psync_sql_free_result(res);
if (unlikely(psync_crc32c(PSYNC_CRC_INITIAL, page->page, page->size) != if (unlikely(pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, page->size) !=
page->crc)) { page->crc)) {
debug( debug(
D_WARNING, D_WARNING,
"page CRC check failed, dropping page, db CRC %u calculated CRC %u", "page CRC check failed, dropping page, db CRC %u calculated CRC %u",
(unsigned)page->crc, (unsigned)page->crc,
(unsigned)psync_crc32c(PSYNC_CRC_INITIAL, page->page, page->size)); (unsigned)pcrc32c_compute(PSYNC_CRC_INITIAL, page->page, page->size));
psync_pagecache_return_free_page(page); psync_pagecache_return_free_page(page);
} else { } else {
page->type = PAGE_TYPE_READ; page->type = PAGE_TYPE_READ;