From: Kevin Day Date: Thu, 7 Apr 2022 05:49:09 +0000 (-0500) Subject: Update: Improve read program logic to allow for catching interrupts during file read. X-Git-Tag: 0.5.10~231 X-Git-Url: https://git.kevux.org/?a=commitdiff_plain;h=b1dddea0ecf4aecfe0c7965b1b40b2432ce47b8a;p=fll Update: Improve read program logic to allow for catching interrupts during file read. Introduce break points in the read process by establishing an outer block loop. Add the signal check before each block read. There may need to be additional, smaller checks to further reduce how often the signal check is performed within these loops. Make sure to use the file stream read rather than the file read. Calculate the file size and pre-allocate the buffer. Determine the exact size needed to avoid additional reallocations by utilizing the modulus operator. The fss_embedded_read is notably out of day and inconsistent with the rest of the fss read programs. This is only partially updated enough to work with the changes. --- diff --git a/level_3/fss_basic_list_read/c/fss_basic_list_read.c b/level_3/fss_basic_list_read/c/fss_basic_list_read.c index dd929c0d1..c6ef62a1b 100644 --- a/level_3/fss_basic_list_read/c/fss_basic_list_read.c +++ b/level_3/fss_basic_list_read/c/fss_basic_list_read.c @@ -516,7 +516,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -546,25 +548,69 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_basic_list_read_block_max) { + file.size_read = fss_basic_list_read_block_read_large; + size_block = fss_basic_list_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_basic_list_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_increase_by", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); break; } - else if (data.buffer.used > data.files.array[data.files.used].range.start) { + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_basic_list_read_block_max && fll_program_standard_signal_received(main)) { + fss_basic_list_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); + + if (F_status_is_error(status)) { + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } + + break; + } + + if (data.buffer.used > data.files.array[data.files.used].range.start) { data.files.array[data.files.used++].range.stop = data.buffer.used - 1; // This standard is newline sensitive, when appending files to the buffer if the file lacks a final newline then this could break the format for files appended thereafter. @@ -578,9 +624,8 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 1; } - - f_file_stream_close(F_true, &file); } // for f_file_stream_close(F_true, &file); diff --git a/level_3/fss_basic_list_read/c/private-common.h b/level_3/fss_basic_list_read/c/private-common.h index 0b07209f1..a6cf5a4be 100644 --- a/level_3/fss_basic_list_read/c/private-common.h +++ b/level_3/fss_basic_list_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * basic_list_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_basic_list_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_basic_list_read_common_ #define fss_basic_list_read_common_allocation_large_d 2048 #define fss_basic_list_read_common_allocation_small_d 128 + + #define fss_basic_list_read_block_max 16777216 + #define fss_basic_list_read_block_read_small 8192 + #define fss_basic_list_read_block_read_large 65536 #endif // _di_fss_basic_list_read_common_ /** diff --git a/level_3/fss_basic_read/c/fss_basic_read.c b/level_3/fss_basic_read/c/fss_basic_read.c index 691ad899d..ece844d2e 100644 --- a/level_3/fss_basic_read/c/fss_basic_read.c +++ b/level_3/fss_basic_read/c/fss_basic_read.c @@ -523,7 +523,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -552,21 +554,64 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_basic_read_block_max) { + file.size_read = fss_basic_read_block_read_large; + size_block = fss_basic_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_basic_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_basic_read_block_max && fll_program_standard_signal_received(main)) { + fss_basic_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); + + if (F_status_is_error(status)) { + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } @@ -586,12 +631,9 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_basic_read/c/private-common.h b/level_3/fss_basic_read/c/private-common.h index 9ad799255..049f1e9d8 100644 --- a/level_3/fss_basic_read/c/private-common.h +++ b/level_3/fss_basic_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * basic_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_basic_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_basic_read_common_ #define fss_basic_read_common_allocation_large_d 2048 #define fss_basic_read_common_allocation_small_d 128 + + #define fss_basic_read_block_max 16777216 + #define fss_basic_read_block_read_small 8192 + #define fss_basic_read_block_read_large 65536 #endif // _di_fss_basic_read_common_ /** diff --git a/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c b/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c index 3fed9e0ff..442ebf963 100644 --- a/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c +++ b/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c @@ -412,7 +412,10 @@ extern "C" { } if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { - f_array_length_t total = 0; + f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; + f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -428,19 +431,21 @@ extern "C" { main->signal_check = 0; } - f_file_t file = f_file_t_initialize; + file.stream = 0; + file.id = -1; - status = f_file_open(data.argv[main->parameters.remaining.array[i]], 0, &file); + status = f_file_stream_open(data.argv[main->parameters.remaining.array[i]], f_string_empty_s, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_open_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_open_s, fll_error_file_type_file_e); break; } - total = 0; + size_file = 0; + + status = f_file_size_by_id(file.id, &size_file); - status = f_file_size_by_id(file.id, &total); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); @@ -450,7 +455,7 @@ extern "C" { } // Skip past empty files. - if (!total) { + if (!size_file) { if (main->parameters.array[fss_embedded_list_read_parameter_total_e].result == f_console_result_found_e) { fll_print_format("%r%r", main->output.to.stream, f_string_ascii_0_s, f_string_eol_s); } @@ -460,12 +465,51 @@ extern "C" { continue; } - status = f_file_read_until(file, total, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_embedded_list_read_block_max) { + file.size_read = fss_embedded_list_read_block_read_large; + size_block = fss_embedded_list_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_embedded_list_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } + + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_embedded_list_read_block_max && fll_program_standard_signal_received(main)) { + fss_embedded_list_read_print_signal_received(&data); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } diff --git a/level_3/fss_embedded_list_read/c/private-common.h b/level_3/fss_embedded_list_read/c/private-common.h index 4f9ee773a..68f5eee97 100644 --- a/level_3/fss_embedded_list_read/c/private-common.h +++ b/level_3/fss_embedded_list_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * embedded_list_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_embedded_list_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_embedded_list_read_common_ #define fss_embedded_list_read_common_allocation_large_d 2048 #define fss_embedded_list_read_common_allocation_small_d 128 + + #define fss_embedded_list_read_block_max 16777216 + #define fss_embedded_list_read_block_read_small 8192 + #define fss_embedded_list_read_block_read_large 65536 #endif // _di_fss_embedded_list_read_common_ /** diff --git a/level_3/fss_extended_list_read/c/fss_extended_list_read.c b/level_3/fss_extended_list_read/c/fss_extended_list_read.c index e3a8416b7..1c49aacc2 100644 --- a/level_3/fss_extended_list_read/c/fss_extended_list_read.c +++ b/level_3/fss_extended_list_read/c/fss_extended_list_read.c @@ -523,7 +523,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -552,25 +554,69 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_extended_list_read_block_max) { + file.size_read = fss_extended_list_read_block_read_large; + size_block = fss_extended_list_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_extended_list_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } + + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_extended_list_read_block_max && fll_program_standard_signal_received(main)) { + fss_extended_list_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } - else if (data.buffer.used > data.files.array[data.files.used].range.start) { + + if (data.buffer.used > data.files.array[data.files.used].range.start) { data.files.array[data.files.used].name = data.argv[main->parameters.remaining.array[i]]; data.files.array[data.files.used++].range.stop = data.buffer.used - 1; @@ -585,12 +631,9 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_extended_list_read/c/private-common.h b/level_3/fss_extended_list_read/c/private-common.h index 18cde8de4..914cae620 100644 --- a/level_3/fss_extended_list_read/c/private-common.h +++ b/level_3/fss_extended_list_read/c/private-common.h @@ -19,10 +19,19 @@ extern "C" { * extended_list_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_extended_list_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_extended_list_read_common_ #define fss_extended_list_read_common_allocation_large_d 2048 #define fss_extended_list_read_common_allocation_small_d 128 + + #define fss_extended_list_read_block_max 16777216 + #define fss_extended_list_read_block_read_small 8192 + #define fss_extended_list_read_block_read_large 65536 #endif // _di_fss_extended_list_read_common_ /** diff --git a/level_3/fss_extended_read/c/fss_extended_read.c b/level_3/fss_extended_read/c/fss_extended_read.c index 081eb89e9..822e3f543 100644 --- a/level_3/fss_extended_read/c/fss_extended_read.c +++ b/level_3/fss_extended_read/c/fss_extended_read.c @@ -522,7 +522,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -551,21 +553,64 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_extended_read_block_max) { + file.size_read = fss_extended_read_block_read_large; + size_block = fss_extended_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_extended_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_extended_read_block_max && fll_program_standard_signal_received(main)) { + fss_extended_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); + + if (F_status_is_error(status)) { + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } @@ -585,12 +630,9 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_extended_read/c/private-common.h b/level_3/fss_extended_read/c/private-common.h index ba363e0b2..a7f642469 100644 --- a/level_3/fss_extended_read/c/private-common.h +++ b/level_3/fss_extended_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * extended_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_extended_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_extended_read_common_ #define fss_extended_read_common_allocation_large_d 2048 #define fss_extended_read_common_allocation_small_d 128 + + #define fss_extended_read_block_max 16777216 + #define fss_extended_read_block_read_small 8192 + #define fss_extended_read_block_read_large 65536 #endif // _di_fss_extended_read_common_ /** diff --git a/level_3/fss_payload_read/c/fss_payload_read.c b/level_3/fss_payload_read/c/fss_payload_read.c index fd682578a..cb46c350f 100644 --- a/level_3/fss_payload_read/c/fss_payload_read.c +++ b/level_3/fss_payload_read/c/fss_payload_read.c @@ -554,7 +554,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; const f_array_length_t buffer_used = data.buffer.used; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -584,48 +586,78 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - // This standard is newline sensitive, when appending files to the buffer if the file lacks a final newline then this could break the format for files appended thereafter. - // Guarantee that a newline exists at the end of the buffer. - // This is done as a pre-process on the next file because the "payload" must always be last and must not have a newline appended. - if (buffer_used != data.buffer.used) { - status = f_string_dynamic_append_assure(f_string_eol_s, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_payload_read_block_max) { + file.size_read = fss_payload_read_block_read_large; + size_block = fss_payload_read_block_max; - if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_string_append_assure", F_true, f_string_ascii_minus_s, f_file_operation_read_s, fll_error_file_type_pipe_e); - } + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_payload_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } + + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; } - status = f_file_stream_read(file, &data.buffer); + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_payload_read_block_max && fll_program_standard_signal_received(main)) { + fss_payload_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } - else if (data.buffer.used > data.files.array[data.files.used].range.start) { + + if (data.buffer.used > data.files.array[data.files.used].range.start) { data.files.array[data.files.used].name = data.argv[main->parameters.remaining.array[i]]; data.files.array[data.files.used++].range.stop = data.buffer.used - 1; } } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_payload_read/c/private-common.h b/level_3/fss_payload_read/c/private-common.h index 49006341f..9a6c75bec 100644 --- a/level_3/fss_payload_read/c/private-common.h +++ b/level_3/fss_payload_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * payload_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_payload_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_payload_read_common_ #define fss_payload_read_common_allocation_large_d 2048 #define fss_payload_read_common_allocation_small_d 128 + + #define fss_payload_read_block_max 16777216 + #define fss_payload_read_block_read_small 8192 + #define fss_payload_read_block_read_large 65536 #endif // _di_fss_payload_read_common_ /** diff --git a/level_3/iki_read/c/iki_read.c b/level_3/iki_read/c/iki_read.c index 5241a50ec..a231a8eba 100644 --- a/level_3/iki_read/c/iki_read.c +++ b/level_3/iki_read/c/iki_read.c @@ -365,6 +365,9 @@ extern "C" { if (F_status_is_fine(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; + f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -380,17 +383,19 @@ extern "C" { main->signal_check = 0; } - macro_f_file_t_reset(file); - file.size_read = 0; + file.stream = 0; + file.id = -1; - status = f_file_open(data.argv[main->parameters.remaining.array[i]], 0, &file); + status = f_file_stream_open(data.argv[main->parameters.remaining.array[i]], f_string_empty_s, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); break; } + size_file = 0; + status = f_file_size_by_id(file.id, &file.size_read); if (F_status_is_error(status)) { @@ -408,14 +413,51 @@ extern "C" { continue; } - ++file.size_read; + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > iki_read_block_max) { + file.size_read = iki_read_block_read_large; + size_block = iki_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = iki_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } - status = f_file_read(file, &data.buffer); + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > iki_read_block_max && fll_program_standard_signal_received(main)) { + iki_read_print_signal_received(&data); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + } break; } diff --git a/level_3/iki_read/c/private-common.h b/level_3/iki_read/c/private-common.h index 7c5536fd3..bf7b048c5 100644 --- a/level_3/iki_read/c/private-common.h +++ b/level_3/iki_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * iki_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * iki_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_iki_read_common_ #define iki_read_common_allocation_large_d 256 #define iki_read_common_allocation_small_d 16 + + #define iki_read_block_max 16777216 + #define iki_read_block_read_small 8192 + #define iki_read_block_read_large 65536 #endif // _di_iki_read_common_ /**