From b1dddea0ecf4aecfe0c7965b1b40b2432ce47b8a Mon Sep 17 00:00:00 2001 From: Kevin Day Date: Thu, 7 Apr 2022 00:49:09 -0500 Subject: [PATCH] Update: Improve read program logic to allow for catching interrupts during file read. Introduce break points in the read process by establishing an outer block loop. Add the signal check before each block read. There may need to be additional, smaller checks to further reduce how often the signal check is performed within these loops. Make sure to use the file stream read rather than the file read. Calculate the file size and pre-allocate the buffer. Determine the exact size needed to avoid additional reallocations by utilizing the modulus operator. The fss_embedded_read is notably out of day and inconsistent with the rest of the fss read programs. This is only partially updated enough to work with the changes. --- .../fss_basic_list_read/c/fss_basic_list_read.c | 57 +++++++++++++++++-- level_3/fss_basic_list_read/c/private-common.h | 9 +++ level_3/fss_basic_read/c/fss_basic_read.c | 56 ++++++++++++++++--- level_3/fss_basic_read/c/private-common.h | 9 +++ .../c/fss_embedded_list_read.c | 62 ++++++++++++++++++--- level_3/fss_embedded_list_read/c/private-common.h | 9 +++ .../c/fss_extended_list_read.c | 59 +++++++++++++++++--- level_3/fss_extended_list_read/c/private-common.h | 9 +++ level_3/fss_extended_read/c/fss_extended_read.c | 56 ++++++++++++++++--- level_3/fss_extended_read/c/private-common.h | 9 +++ level_3/fss_payload_read/c/fss_payload_read.c | 64 ++++++++++++++++------ level_3/fss_payload_read/c/private-common.h | 9 +++ level_3/iki_read/c/iki_read.c | 56 ++++++++++++++++--- level_3/iki_read/c/private-common.h | 9 +++ 14 files changed, 413 insertions(+), 60 deletions(-) diff --git a/level_3/fss_basic_list_read/c/fss_basic_list_read.c b/level_3/fss_basic_list_read/c/fss_basic_list_read.c index dd929c0..c6ef62a 100644 --- a/level_3/fss_basic_list_read/c/fss_basic_list_read.c +++ b/level_3/fss_basic_list_read/c/fss_basic_list_read.c @@ -516,7 +516,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -546,25 +548,69 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_basic_list_read_block_max) { + file.size_read = fss_basic_list_read_block_read_large; + size_block = fss_basic_list_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_basic_list_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_increase_by", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); break; } - else if (data.buffer.used > data.files.array[data.files.used].range.start) { + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_basic_list_read_block_max && fll_program_standard_signal_received(main)) { + fss_basic_list_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); + + if (F_status_is_error(status)) { + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } + + break; + } + + if (data.buffer.used > data.files.array[data.files.used].range.start) { data.files.array[data.files.used++].range.stop = data.buffer.used - 1; // This standard is newline sensitive, when appending files to the buffer if the file lacks a final newline then this could break the format for files appended thereafter. @@ -578,9 +624,8 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 1; } - - f_file_stream_close(F_true, &file); } // for f_file_stream_close(F_true, &file); diff --git a/level_3/fss_basic_list_read/c/private-common.h b/level_3/fss_basic_list_read/c/private-common.h index 0b07209..a6cf5a4 100644 --- a/level_3/fss_basic_list_read/c/private-common.h +++ b/level_3/fss_basic_list_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * basic_list_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_basic_list_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_basic_list_read_common_ #define fss_basic_list_read_common_allocation_large_d 2048 #define fss_basic_list_read_common_allocation_small_d 128 + + #define fss_basic_list_read_block_max 16777216 + #define fss_basic_list_read_block_read_small 8192 + #define fss_basic_list_read_block_read_large 65536 #endif // _di_fss_basic_list_read_common_ /** diff --git a/level_3/fss_basic_read/c/fss_basic_read.c b/level_3/fss_basic_read/c/fss_basic_read.c index 691ad89..ece844d 100644 --- a/level_3/fss_basic_read/c/fss_basic_read.c +++ b/level_3/fss_basic_read/c/fss_basic_read.c @@ -523,7 +523,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -552,21 +554,64 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_basic_read_block_max) { + file.size_read = fss_basic_read_block_read_large; + size_block = fss_basic_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_basic_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_basic_read_block_max && fll_program_standard_signal_received(main)) { + fss_basic_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); + + if (F_status_is_error(status)) { + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } @@ -586,12 +631,9 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_basic_read/c/private-common.h b/level_3/fss_basic_read/c/private-common.h index 9ad7992..049f1e9 100644 --- a/level_3/fss_basic_read/c/private-common.h +++ b/level_3/fss_basic_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * basic_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_basic_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_basic_read_common_ #define fss_basic_read_common_allocation_large_d 2048 #define fss_basic_read_common_allocation_small_d 128 + + #define fss_basic_read_block_max 16777216 + #define fss_basic_read_block_read_small 8192 + #define fss_basic_read_block_read_large 65536 #endif // _di_fss_basic_read_common_ /** diff --git a/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c b/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c index 3fed9e0..442ebf9 100644 --- a/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c +++ b/level_3/fss_embedded_list_read/c/fss_embedded_list_read.c @@ -412,7 +412,10 @@ extern "C" { } if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { - f_array_length_t total = 0; + f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; + f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -428,19 +431,21 @@ extern "C" { main->signal_check = 0; } - f_file_t file = f_file_t_initialize; + file.stream = 0; + file.id = -1; - status = f_file_open(data.argv[main->parameters.remaining.array[i]], 0, &file); + status = f_file_stream_open(data.argv[main->parameters.remaining.array[i]], f_string_empty_s, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_open_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_open_s, fll_error_file_type_file_e); break; } - total = 0; + size_file = 0; + + status = f_file_size_by_id(file.id, &size_file); - status = f_file_size_by_id(file.id, &total); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); @@ -450,7 +455,7 @@ extern "C" { } // Skip past empty files. - if (!total) { + if (!size_file) { if (main->parameters.array[fss_embedded_list_read_parameter_total_e].result == f_console_result_found_e) { fll_print_format("%r%r", main->output.to.stream, f_string_ascii_0_s, f_string_eol_s); } @@ -460,12 +465,51 @@ extern "C" { continue; } - status = f_file_read_until(file, total, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_embedded_list_read_block_max) { + file.size_read = fss_embedded_list_read_block_read_large; + size_block = fss_embedded_list_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_embedded_list_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } + + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_embedded_list_read_block_max && fll_program_standard_signal_received(main)) { + fss_embedded_list_read_print_signal_received(&data); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } diff --git a/level_3/fss_embedded_list_read/c/private-common.h b/level_3/fss_embedded_list_read/c/private-common.h index 4f9ee77..68f5eee 100644 --- a/level_3/fss_embedded_list_read/c/private-common.h +++ b/level_3/fss_embedded_list_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * embedded_list_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_embedded_list_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_embedded_list_read_common_ #define fss_embedded_list_read_common_allocation_large_d 2048 #define fss_embedded_list_read_common_allocation_small_d 128 + + #define fss_embedded_list_read_block_max 16777216 + #define fss_embedded_list_read_block_read_small 8192 + #define fss_embedded_list_read_block_read_large 65536 #endif // _di_fss_embedded_list_read_common_ /** diff --git a/level_3/fss_extended_list_read/c/fss_extended_list_read.c b/level_3/fss_extended_list_read/c/fss_extended_list_read.c index e3a8416..1c49aac 100644 --- a/level_3/fss_extended_list_read/c/fss_extended_list_read.c +++ b/level_3/fss_extended_list_read/c/fss_extended_list_read.c @@ -523,7 +523,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -552,25 +554,69 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_extended_list_read_block_max) { + file.size_read = fss_extended_list_read_block_read_large; + size_block = fss_extended_list_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_extended_list_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } + + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_extended_list_read_block_max && fll_program_standard_signal_received(main)) { + fss_extended_list_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } - else if (data.buffer.used > data.files.array[data.files.used].range.start) { + + if (data.buffer.used > data.files.array[data.files.used].range.start) { data.files.array[data.files.used].name = data.argv[main->parameters.remaining.array[i]]; data.files.array[data.files.used++].range.stop = data.buffer.used - 1; @@ -585,12 +631,9 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_extended_list_read/c/private-common.h b/level_3/fss_extended_list_read/c/private-common.h index 18cde8d..914cae6 100644 --- a/level_3/fss_extended_list_read/c/private-common.h +++ b/level_3/fss_extended_list_read/c/private-common.h @@ -19,10 +19,19 @@ extern "C" { * extended_list_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_extended_list_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_extended_list_read_common_ #define fss_extended_list_read_common_allocation_large_d 2048 #define fss_extended_list_read_common_allocation_small_d 128 + + #define fss_extended_list_read_block_max 16777216 + #define fss_extended_list_read_block_read_small 8192 + #define fss_extended_list_read_block_read_large 65536 #endif // _di_fss_extended_list_read_common_ /** diff --git a/level_3/fss_extended_read/c/fss_extended_read.c b/level_3/fss_extended_read/c/fss_extended_read.c index 081eb89..822e3f5 100644 --- a/level_3/fss_extended_read/c/fss_extended_read.c +++ b/level_3/fss_extended_read/c/fss_extended_read.c @@ -522,7 +522,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -551,21 +553,64 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - status = f_file_stream_read(file, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_extended_read_block_max) { + file.size_read = fss_extended_read_block_read_large; + size_block = fss_extended_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_extended_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_extended_read_block_max && fll_program_standard_signal_received(main)) { + fss_extended_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); + + if (F_status_is_error(status)) { + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } @@ -585,12 +630,9 @@ extern "C" { } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_extended_read/c/private-common.h b/level_3/fss_extended_read/c/private-common.h index ba363e0..a7f6424 100644 --- a/level_3/fss_extended_read/c/private-common.h +++ b/level_3/fss_extended_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * extended_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_extended_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_extended_read_common_ #define fss_extended_read_common_allocation_large_d 2048 #define fss_extended_read_common_allocation_small_d 128 + + #define fss_extended_read_block_max 16777216 + #define fss_extended_read_block_read_small 8192 + #define fss_extended_read_block_read_large 65536 #endif // _di_fss_extended_read_common_ /** diff --git a/level_3/fss_payload_read/c/fss_payload_read.c b/level_3/fss_payload_read/c/fss_payload_read.c index fd68257..cb46c35 100644 --- a/level_3/fss_payload_read/c/fss_payload_read.c +++ b/level_3/fss_payload_read/c/fss_payload_read.c @@ -554,7 +554,9 @@ extern "C" { if (F_status_is_error_not(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; f_array_length_t size_file = 0; + f_array_length_t size_read = 0; const f_array_length_t buffer_used = data.buffer.used; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -584,48 +586,78 @@ extern "C" { } size_file = 0; + status = f_file_size_by_id(file.id, &size_file); if (F_status_is_error(status)) { fll_error_file_print(main->error, F_status_set_fine(status), "f_file_size_by_id", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + f_file_stream_close(F_true, &file); + break; } if (size_file) { - file.size_read = size_file + 1; - // This standard is newline sensitive, when appending files to the buffer if the file lacks a final newline then this could break the format for files appended thereafter. - // Guarantee that a newline exists at the end of the buffer. - // This is done as a pre-process on the next file because the "payload" must always be last and must not have a newline appended. - if (buffer_used != data.buffer.used) { - status = f_string_dynamic_append_assure(f_string_eol_s, &data.buffer); + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > fss_payload_read_block_max) { + file.size_read = fss_payload_read_block_read_large; + size_block = fss_payload_read_block_max; - if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_string_append_assure", F_true, f_string_ascii_minus_s, f_file_operation_read_s, fll_error_file_type_pipe_e); - } + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = fss_payload_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } + + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; } - status = f_file_stream_read(file, &data.buffer); + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > fss_payload_read_block_max && fll_program_standard_signal_received(main)) { + fss_payload_read_print_signal_received(main); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for + + f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_read_s, fll_error_file_type_file_e); + } break; } - else if (data.buffer.used > data.files.array[data.files.used].range.start) { + + if (data.buffer.used > data.files.array[data.files.used].range.start) { data.files.array[data.files.used].name = data.argv[main->parameters.remaining.array[i]]; data.files.array[data.files.used++].range.stop = data.buffer.used - 1; } } else { data.files.array[data.files.used].range.start = 1; + data.files.array[data.files.used].range.stop = 0; } - - f_file_stream_close(F_true, &file); } // for - - f_file_stream_close(F_true, &file); } if (F_status_is_error_not(status)) { diff --git a/level_3/fss_payload_read/c/private-common.h b/level_3/fss_payload_read/c/private-common.h index 4900634..9a6c75b 100644 --- a/level_3/fss_payload_read/c/private-common.h +++ b/level_3/fss_payload_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * payload_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * fss_payload_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_fss_payload_read_common_ #define fss_payload_read_common_allocation_large_d 2048 #define fss_payload_read_common_allocation_small_d 128 + + #define fss_payload_read_block_max 16777216 + #define fss_payload_read_block_read_small 8192 + #define fss_payload_read_block_read_large 65536 #endif // _di_fss_payload_read_common_ /** diff --git a/level_3/iki_read/c/iki_read.c b/level_3/iki_read/c/iki_read.c index 5241a50..a231a8e 100644 --- a/level_3/iki_read/c/iki_read.c +++ b/level_3/iki_read/c/iki_read.c @@ -365,6 +365,9 @@ extern "C" { if (F_status_is_fine(status) && main->parameters.remaining.used > 0) { f_file_t file = f_file_t_initialize; + f_array_length_t size_block = 0; + f_array_length_t size_file = 0; + f_array_length_t size_read = 0; for (f_array_length_t i = 0; i < main->parameters.remaining.used; ++i) { @@ -380,17 +383,19 @@ extern "C" { main->signal_check = 0; } - macro_f_file_t_reset(file); - file.size_read = 0; + file.stream = 0; + file.id = -1; - status = f_file_open(data.argv[main->parameters.remaining.array[i]], 0, &file); + status = f_file_stream_open(data.argv[main->parameters.remaining.array[i]], f_string_empty_s, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_open", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); break; } + size_file = 0; + status = f_file_size_by_id(file.id, &file.size_read); if (F_status_is_error(status)) { @@ -408,14 +413,51 @@ extern "C" { continue; } - ++file.size_read; + // Enforce a max block read size to allow for interrupts to be processed beteween blocks. + if (size_file > iki_read_block_max) { + file.size_read = iki_read_block_read_large; + size_block = iki_read_block_max; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + (size_block - (size_file % size_block)) + 1, &data.buffer); + } + else { + file.size_read = iki_read_block_read_small; + size_block = size_file; + + // Pre-allocate entire file buffer plus space for the terminating NULL. + f_string_dynamic_increase_by(size_file + 1, &data.buffer); + } - status = f_file_read(file, &data.buffer); + if (F_status_is_error(status)) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_string_dynamic_resize", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + + f_file_stream_close(F_true, &file); + + break; + } + + for (size_read = 0; size_read < size_file; size_read += size_block) { + + // The signal check is always performed on each pass. + if (size_file > iki_read_block_max && fll_program_standard_signal_received(main)) { + iki_read_print_signal_received(&data); + + status = F_status_set_error(F_interrupt); + + break; + } + + status = f_file_stream_read_until(file, size_block, &data.buffer); + if (F_status_is_error(status)) break; + } // for f_file_stream_close(F_true, &file); if (F_status_is_error(status)) { - fll_error_file_print(main->error, F_status_set_fine(status), "f_file_read", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + if (F_status_set_fine(status) != F_interrupt) { + fll_error_file_print(main->error, F_status_set_fine(status), "f_file_stream_read_until", F_true, data.argv[main->parameters.remaining.array[i]], f_file_operation_process_s, fll_error_file_type_file_e); + } break; } diff --git a/level_3/iki_read/c/private-common.h b/level_3/iki_read/c/private-common.h index 7c5536f..bf7b048 100644 --- a/level_3/iki_read/c/private-common.h +++ b/level_3/iki_read/c/private-common.h @@ -18,10 +18,19 @@ extern "C" { * iki_read_common_allocation_*: * - large: An allocation step used for buffers that are anticipated to have large buffers. * - small: An allocation step used for buffers that are anticipated to have small buffers. + * + * iki_read_block_*: + * - max: The max block read size before checking for interrupt. + * - read_small: The block read size for small files. + * - read_large: The block read size for large files. */ #ifndef _di_iki_read_common_ #define iki_read_common_allocation_large_d 256 #define iki_read_common_allocation_small_d 16 + + #define iki_read_block_max 16777216 + #define iki_read_block_read_small 8192 + #define iki_read_block_read_large 65536 #endif // _di_iki_read_common_ /** -- 1.8.3.1