fix archive and directory recursion errors

This commit is contained in:
Peter McGoron 2023-12-12 15:18:43 -05:00
parent 95facd49ae
commit bfc99264dd
2 changed files with 126 additions and 68 deletions

View File

@ -1 +1 @@
tcc\tcc.exe -Wall -o wlb.exe wlb.c sha-256.c sqlite3.c -DSQLITE_OMIT_LOAD_EXTENSION
tcc\tcc.exe -Wall -o wlb.exe wlb.c sha-256.c -L. -lsqlite3

192
wlb.c
View File

@ -19,7 +19,7 @@
*/
/*************************************************************************
* Compatability defines for non-Unicode systems
* Compatability defines for non-Unicode systems
************************************************************************/
#ifdef UNICODE
@ -27,11 +27,15 @@
# define sqlite3_open_U sqlite3_open16
# define sqlite3_errmsg_U sqlite3_errmsg16
# define sqlite3_bind_text_U sqlite3_bind_text16
# define sqlite3_column_bytes_U sqlite3_column_bytes16
# define sqlite3_column_text_U sqlite3_column_text16
#else
# define ENCODING SQLITE3_UTF8
# define sqlite3_open_U sqlite3_open
# define sqlite3_errmsg_U sqlite3_errmsg
# define sqlite3_bind_text_U sqlite3_bind_text
# define sqlite3_column_bytes_U sqlite3_column_bytes
# define sqlite3_column_text_U sqlite3_column_text
#endif
/****************
@ -39,20 +43,24 @@
***************/
sqlite3 *g_db = NULL;
int g_verbose = 0;
sqlite_int64 g_backup_id = -1;
enum {
NORMAL_MODE,
VERBOSE_MODE,
DEBUG_MODE
} g_verbose = NORMAL_MODE;
sqlite3_int64 g_backup_id = -1;
/********************
* Utility Functions
*******************/
static void _log(TCHAR *msg, ...)
static void _log(TCHAR *msg, int verb, ...)
{
va_list va;
if (g_verbose) {
va_start(va, msg);
_vtprintf(msg, va);
if (g_verbose >= verb) {
va_start(va, verb);
_vftprintf(stderr, msg, va);
va_end(va);
}
}
@ -79,28 +87,29 @@ static void _die(TCHAR *emsg, ...)
/* 'wlb' in ASCII */
#define UPPER_VERSION 0x776c62
#define CUR_VERSION 0x776c6201
#define S(s) #s
#define INIT_SCRIPT "\
BEGIN; \
PRAGMA user_version = " S(CUR_VERSION) ";\
CREATE TABLE chunks (\
rowid INTEGER PRIMARY KEY, \
sha256 TEXT UNIQUE NOT NULL, \
data BLOB NOT NULL\
); \
CREATE TABLE backup_ids (\
ts TEXT UNIQUE NOT NULL,\
rowid INTEGER PRIMARY KEY\
); \
CREATE TABLE backups (\
backup INTEGER NOT NULL REFERENCES backup_ids ON DELETE CASCADE, \
path TEXT NOT NULL, \
chunk INTEGER NOT NULL REFERENCES chunks ON DELETE RESTRICT\
); \
CREATE INDEX backups_paths ON backups (path); \
CREATE INDEX backups_backup ON backups (backup); \
CREATE INDEX backups_chunk ON backups (chunk);\
COMMIT;"
#define S2(s) #s
#define S(s) S2(s)
static const char *init_script =
"BEGIN;\n"
"PRAGMA user_version = " S(CUR_VERSION) ";\n"
"CREATE TABLE chunks (\n"
" rowid INTEGER PRIMARY KEY, \n"
" sha256 TEXT UNIQUE NOT NULL, \n"
" data BLOB NOT NULL\n"
"); \n"
"CREATE TABLE backup_ids (\n"
" ts TEXT UNIQUE NOT NULL,\n"
" rowid INTEGER PRIMARY KEY\n"
"); \n"
"CREATE TABLE backups (\n"
" backup INTEGER NOT NULL REFERENCES backup_ids ON DELETE CASCADE, \n"
" path TEXT NOT NULL, \n"
" chunk INTEGER NOT NULL REFERENCES chunks ON DELETE RESTRICT\n"
"); \n"
"CREATE INDEX backups_paths ON backups (path); \n"
"CREATE INDEX backups_backup ON backups (backup); \n"
"CREATE INDEX backups_chunk ON backups (chunk);\n"
"COMMIT;";
/* Check if the database version is correct. If it is not correct
* and the create flag is enabled, then initialize a new database.
@ -112,10 +121,11 @@ static void check_db_version(int did_not_exist)
/* DB did not exist beforehand: initialize DB */
if (did_not_exist) {
if (sqlite3_exec(g_db, INIT_SCRIPT, NULL, NULL, NULL) != SQLITE_OK)
log("Writing init script %s\n", DEBUG_MODE, init_script);
if (sqlite3_exec(g_db, init_script, NULL, NULL, NULL) != SQLITE_OK)
die("Error initializing database: %s\n",
sqlite3_errmsg_U(g_db));
log("%s", "Intialized database\n");
log("%s", VERBOSE_MODE, "Intialized database\n");
return;
}
@ -144,7 +154,7 @@ static void initialize_backup_id(void)
sqlite3_stmt *stmt;
if (sqlite3_prepare(g_db,
"INSERT INTO backup_ids (ts) VALUES (datetime()) RETURNING rowid;",
"INSERT INTO backup_ids (ts) VALUES (datetime('subsec')) RETURNING ts,rowid;",
-1, &stmt, NULL) != SQLITE_OK) {
die("failed to prepare inserting timestamp: %s\n",
sqlite3_errmsg_U(g_db));
@ -153,7 +163,10 @@ static void initialize_backup_id(void)
if (sqlite3_step(stmt) != SQLITE_ROW)
die("failed to insert timestamp: %s\n",
sqlite3_errmsg_U(g_db));
g_backup_id = sqlite3_column_int64(stmt, 0);
log("Timestamp: %s\n", NORMAL_MODE, sqlite3_column_text_U(stmt, 0));
g_backup_id = sqlite3_column_int64(stmt, 1);
sqlite3_finalize(stmt);
}
@ -164,14 +177,15 @@ static void open_db(TCHAR *fn)
== INVALID_FILE_ATTRIBUTES;
char *errmsg;
log("Opening DB %s\n", DEBUG_MODE, fn);
if (sqlite3_open_U(fn, &g_db) != SQLITE_OK)
die("Error opening database %s: %s\n", fn,
sqlite3_errmsg_U(g_db));
check_db_version(did_not_exist);
/* Store current timestamp */
if (sqlite3_exec(g_db, "BEGIN;", NULL, NULL, &errmsg) != SQLITE_OK)
die("failed to begin transaction: %s\n", errmsg);
log("Started transaction\n", DEBUG_MODE);
}
/* Insert a backup record into the database.
@ -184,39 +198,42 @@ static int insert_backup_record(TCHAR *name, sqlite_int64 chunk_id)
sqlite3_stmt *stmt;
int r = 0;
log("Inserting backup record (%s, %lld)\n", DEBUG_MODE, name, (long long) chunk_id);
if (sqlite3_prepare(g_db,
"INSERT INTO backups (backup, path, chunk) VALUES (?,?,?);",
-1, &stmt, NULL) != SQLITE_OK) {
_tprintf("Could not prepare backup insert statement for %s: %s\n",
name, sqlite3_errmsg_U(g_db));
log("Could not prepare backup insert statement for %s: %s\n",
NORMAL_MODE, name, sqlite3_errmsg_U(g_db));
return 0;
}
if (sqlite3_bind_int64(stmt, 1, g_backup_id) != SQLITE_OK) {
_tprintf("Could not bind backup id for %s: %s\n", name,
sqlite3_errmsg_U(g_db));
log("Could not bind backup id for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto end;
}
if (sqlite3_bind_text_U(stmt, 2, name, -1, SQLITE_TRANSIENT) != SQLITE_OK) {
_tprintf("Could not bind path for %s: %s\n", name,
sqlite3_errmsg_U(g_db));
log("Could not bind path for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto end;
}
if (sqlite3_bind_int64(stmt, 3, chunk_id) != SQLITE_OK) {
_tprintf("Could not bind chunk for %s: %s\n", name,
log("Could not bind chunk for %s: %s\n", NORMAL_MODE, name,
sqlite3_errmsg_U(g_db));
goto end;
}
if (sqlite3_step(stmt) != SQLITE_DONE) {
_tprintf("Error in inserting chunk for %s: %s\n", name,
log("Error in inserting chunk for %s: %s\n", NORMAL_MODE, name,
sqlite3_errmsg_U(g_db));
goto end;
}
r = 1;
log("Sucessful insertion\n", DEBUG_MODE);
end:
sqlite3_finalize(stmt);
return r;
@ -234,33 +251,37 @@ static int write_chunk(TCHAR *name, HANDLE f, sqlite_int64 rowid)
int has_written = 0;
int r = 0;
log("Writing chunk for %s, rowid=%lld\n", DEBUG_MODE, name, (long long)rowid);
if (sqlite3_blob_open(g_db, "main", "chunks", "data", rowid, 1,
&blob) != SQLITE_OK) {
_tprintf("Could not open BLOB to write chunk for %s: %s\n",
log("Could not open BLOB to write chunk for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
return 0;
}
for (;;) {
if (ReadFile(f, buf, sizeof(buf), &read, NULL) == FALSE) {
_tprintf("failed to read in %s for sha256 calculation\n",
log("failed to read in %s for sha256 calculation\n", NORMAL_MODE,
name);
goto end;
}
if (read == 0)
break;
if (sqlite3_blob_write(g_db, buf, read, has_written)
if (sqlite3_blob_write(blob, buf, read, has_written)
!= SQLITE_OK) {
_tprintf("Could not write to BLOB for %s: %s\n",
log("Could not write to BLOB for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto end;
}
has_written += read;
log("Blob written %d bytes\n", DEBUG_MODE, has_written);
}
r = 1;
log("Succesfully written blob\n", DEBUG_MODE);
end:
sqlite3_blob_close(blob);
return r;
@ -282,16 +303,17 @@ static int insert_chunk(TCHAR *name, HANDLE f, sqlite_int64 *rowid,
if (SetFilePointer(f, 0, NULL, FILE_BEGIN)
== INVALID_SET_FILE_POINTER) {
_tprintf("rewind for %s failed\n", name);
log("rewind for %s failed\n", NORMAL_MODE, name);
return 0;
}
fsizeLow = GetFileSize(f, &fsizeHigh);
if (fsizeLow == INVALID_FILE_SIZE) {
_tprintf("Could not get the file size of %s\n", name);
log("Could not get the file size of %s\n", NORMAL_MODE, name);
return 0;
}
fsize = fsizeLow | (fsizeHigh << 32);
log("File size: %lld\n", (long long)fsize);
/* Chunks are fixed size in SQLite. They need to be pre-allocated
* with the size of a file before a file can be written to it.
@ -299,37 +321,48 @@ static int insert_chunk(TCHAR *name, HANDLE f, sqlite_int64 *rowid,
if (sqlite3_prepare(g_db,
"INSERT INTO chunks (sha256, data) VALUES (hex(?), zeroblob(?)) RETURNING rowid;",
-1, &stmt, NULL) != SQLITE_OK) {
_tprintf("Could not prepare chunk insertion for %s: %s\n",
log("Could not prepare chunk insertion for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
return 0;
}
if (sqlite3_bind_blob(stmt, 1, sha256, SHASIZ, SQLITE_TRANSIENT)
!= SQLITE_OK) {
_tprintf("Could not bind sha256 to statement for %s: %s\n",
log("Could not bind sha256 to statement for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto finalize;
}
if (sqlite3_bind_int64(stmt, 2, fsize) != SQLITE_OK) {
_tprintf("Could not bind file size to statement for %s: %s\n",
log("Could not bind file size to statement for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto finalize;
}
if (sqlite3_step(stmt) != SQLITE_ROW) {
_tprintf("Could not step chunk insertion statement for %s: %s\n",
log("Could not step chunk insertion statement for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto finalize;
}
*rowid = sqlite3_column_int64(stmt, 0);
log("Allocated new chunk at %lld\n", DEBUG_MODE, (long long)*rowid);
r = write_chunk(name, f, *rowid);
finalize:
sqlite3_finalize(stmt);
return r;
}
/* Print SHA256 sum ASCII representation to terminal (debug mode only) */
static void dump_sha256(uint8_t sha256[SHASIZ])
{
int i;
for (i = 0; i < SHASIZ; i++) {
log("%hhX", DEBUG_MODE, sha256[i]);
}
}
/* Check if the SHA256 checksum already exists in the database.
*
* If the checksum exists, ``rowid`` is filled with the row in ``chunks``
@ -350,14 +383,14 @@ static int check_sha256(TCHAR *name, uint8_t sha256[SHASIZ],
if (sqlite3_prepare_v2(g_db,
"SELECT rowid FROM chunks WHERE sha256 = hex(?);",
-1, &stmt, NULL) != SQLITE_OK) {
_tprintf("failed to prepare sha256 statement for %s: %s\n",
log("failed to prepare sha256 statement for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto end;
}
if (sqlite3_bind_blob(stmt, 1, sha256, SHASIZ, SQLITE_TRANSIENT)
!= SQLITE_OK) {
_tprintf("failed to bind sha256 value for %s: %s\n",
log("failed to bind sha256 value for %s: %s\n", NORMAL_MODE,
name, sqlite3_errmsg_U(g_db));
goto end;
}
@ -375,6 +408,7 @@ static int check_sha256(TCHAR *name, uint8_t sha256[SHASIZ],
r = 1;
end:
log("rowid: %lld\n", DEBUG_MODE, (long long)*rowid);
sqlite3_finalize(stmt);
return r;
}
@ -394,7 +428,7 @@ static int calculate_sha256(TCHAR *name, HANDLE f, uint8_t sha256[SHASIZ])
sha_256_init(&sha_state, sha256);
for (;;) {
if (ReadFile(f, buf, sizeof(buf), &read, NULL) == FALSE) {
_tprintf("failed to read in %s for sha256 calculation\n",
log("failed to read in %s for sha256 calculation\n", NORMAL_MODE,
name);
return 0;
}
@ -405,6 +439,9 @@ static int calculate_sha256(TCHAR *name, HANDLE f, uint8_t sha256[SHASIZ])
sha_256_write(&sha_state, buf, read);
}
log("SHA256 sum of %s: ", DEBUG_MODE, name);
dump_sha256(sha256);
log("\n", DEBUG_MODE);
sha_256_close(&sha_state);
return 1;
}
@ -437,13 +474,15 @@ static void archive_file(TCHAR *name)
if (chunks_rowid < 0) {
if (!insert_chunk(name, f, &chunks_rowid, sha256))
goto end;
} else {
log("SHA sum already exists\n", DEBUG_MODE);
}
if (!insert_backup_record(name, chunks_rowid))
goto end;
success = 1;
log("Archived %s\n", name);
log("Archived %s\n", VERBOSE_MODE, name);
end:
if (sqlite3_exec(g_db, success ? "RELEASE file;" : "ROLLBACK TO file;", NULL, NULL,
&errstr) != SQLITE_OK)
@ -457,24 +496,37 @@ static void archive_directory(TCHAR *dirname)
WIN32_FIND_DATA fdata;
HANDLE dhandle;
TCHAR pathname[PATH_MAX];
dhandle = FindFirstFile(dirname, &fdata);
if (dhandle == INVALID_HANDLE_VALUE) {
_tprintf("Failed to open directory %s\n", dirname);
if (_sntprintf(pathname, PATH_MAX, TEXT("%s\\*"), dirname) < 0) {
log("Pathname %s\\* too long\n", NORMAL_MODE, dirname);
return;
}
dhandle = FindFirstFile(pathname, &fdata);
if (dhandle == INVALID_HANDLE_VALUE) {
log("Failed to open directory %s\n", NORMAL_MODE, dirname);
return;
}
log("Archiving directory %s\n", VERBOSE_MODE, dirname);
do {
/* Windows versions prior to 11 (!) include _snprintf() as
* a non-standard version of snprintf(). _snprintf() will
* return -1 if the string does not fit in the buffer.
*/
if (_sntprintf(pathname, PATH_MAX, "%s\\%s", dirname,
if (_sntprintf(pathname, PATH_MAX, TEXT("%s\\%s"), dirname,
fdata.cFileName) < 0) {
_tprintf("Pathname for %s\\%s too long\n", dirname,
log("Pathname for %s\\%s too long\n", NORMAL_MODE, dirname,
fdata.cFileName);
continue;
}
if (_tcscmp(fdata.cFileName, TEXT(".")) == 0
|| _tcscmp(fdata.cFileName, TEXT("..")) == 0) {
log("Skipping . or ..\n", DEBUG_MODE);
continue;
}
if (fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
archive_directory(pathname);
else
@ -490,11 +542,11 @@ static void archive_file_or_directory(TCHAR *name)
{
DWORD attr = GetFileAttributes(name);
if (attr == INVALID_FILE_ATTRIBUTES) {
_tprintf(TEXT("failed to open %s\n"), name);
log("failed to open %s\n", NORMAL_MODE, name);
return;
}
if (g_backup_id < 0)
if (g_backup_id == -1)
initialize_backup_id();
if (attr & FILE_ATTRIBUTE_DIRECTORY)
@ -506,8 +558,10 @@ static void archive_file_or_directory(TCHAR *name)
/* Print instructions to console and exit. */
static void usage(void)
{
_tprintf(TEXT("\nwlb [\\V] [\\H] [\\D DBNAME] [\\A DIRECTORIES...]\n"));
_tprintf(TEXT("\nwlb [\\V] [\\VV] [\\S] [\\H] [\\D DBNAME] [\\A DIRECTORIES...]\n"));
_tprintf(TEXT("\\V: Verbose\n"));
_tprintf(TEXT("\\VV: Debug mode\n"));
_tprintf(TEXT("\\S: Go back to normal mode\n"));
_tprintf(TEXT("\\H: Display the help\n"));
_tprintf(TEXT("\\D: Database (create if does not exist)\n"));
_tprintf(TEXT("\\A: Archive directories\n"));
@ -528,7 +582,11 @@ int _tmain(int argc, TCHAR *argv[])
i++;
open_db(argv[i]);
} else if (_tcscmp(argv[i], TEXT("\\V")) == 0) {
g_verbose = 1;
g_verbose = VERBOSE_MODE;
} else if (_tcscmp(argv[i], TEXT("\\VV")) == 0) {
g_verbose = DEBUG_MODE;
} else if (_tcscmp(argv[i], TEXT("\\S")) == 0) {
g_verbose = NORMAL_MODE;
} else {
usage();
}
@ -537,5 +595,5 @@ int _tmain(int argc, TCHAR *argv[])
if (g_db)
sqlite3_exec(g_db, "COMMIT;", NULL, NULL, NULL);
sqlite3_close(g_db);
return 0;
}
return 0;
}