From 5bb09fa1f4eb7d028adfe20ebd1270c8e243bc3b Mon Sep 17 00:00:00 2001 From: Kevin Day Date: Tue, 7 Nov 2023 21:12:09 -0600 Subject: [PATCH] Progress: Further work in TacocaT. --- data/build/tacocat/dependencies | 1 + data/build/tacocat/settings | 2 +- sources/c/tacocat/main/common.c | 5 ++ sources/c/tacocat/main/common/define.h | 12 +++-- sources/c/tacocat/main/common/enumeration.h | 28 +++++----- sources/c/tacocat/main/common/print.h | 4 +- sources/c/tacocat/main/common/type.c | 16 ++++++ sources/c/tacocat/main/common/type.h | 58 ++++++++++++++++++-- sources/c/tacocat/main/receive.c | 6 +-- sources/c/tacocat/main/send.c | 82 +++++++++++++++++++++++++---- sources/c/tacocat/main/send.h | 3 ++ sources/c/tacocat/main/tacocat.h | 2 + 12 files changed, 181 insertions(+), 38 deletions(-) diff --git a/data/build/tacocat/dependencies b/data/build/tacocat/dependencies index 82111db..50c514d 100644 --- a/data/build/tacocat/dependencies +++ b/data/build/tacocat/dependencies @@ -22,6 +22,7 @@ f_thread f_time fl_conversion +fl_fss fl_print fll_error diff --git a/data/build/tacocat/settings b/data/build/tacocat/settings index 03921a7..4bc96ad 100644 --- a/data/build/tacocat/settings +++ b/data/build/tacocat/settings @@ -32,7 +32,7 @@ build_indexer_arguments rcs build_language c build_libraries -lc -build_libraries-individual -lfll_error -lfll_print -lfll_program -lfl_conversion -lfl_print -lf_color -lf_compare -lf_console -lf_conversion -lf_file -lf_fss -lf_memory -lf_network -lf_path -lf_pipe -lf_print -lf_signal -lf_socket -lf_string -lf_time -lf_type_array -lf_utf +build_libraries-individual -lfll_error -lfll_print -lfll_program -lfl_conversion -lfl_fss -lfl_print -lf_color -lf_compare -lf_console -lf_conversion -lf_file -lf_fss -lf_memory -lf_network -lf_path -lf_pipe -lf_print -lf_signal -lf_socket -lf_string -lf_time -lf_type_array -lf_utf build_libraries-individual_thread -lf_thread build_libraries-level -lfll_2 -lfll_1 -lfll_0 build_libraries-monolithic -lfll diff --git a/sources/c/tacocat/main/common.c b/sources/c/tacocat/main/common.c index f205e5f..23e059a 100644 --- a/sources/c/tacocat/main/common.c +++ b/sources/c/tacocat/main/common.c @@ -332,8 +332,11 @@ extern "C" { j = sets[i]->used; sets[i]->array[j].size_block = default_block_size[i]; + sets[i]->array[j].abstruses.used = 0; sets[i]->array[j].buffer.used = 0; + sets[i]->array[j].cache.used = 0; sets[i]->array[j].client.used = 0; + sets[i]->array[j].header.used = 0; sets[i]->array[j].flag = 0; sets[i]->array[j].name.used = 0; sets[i]->array[j].network.used = 0; @@ -343,6 +346,8 @@ extern "C" { sets[i]->array[j].packet.payload.stop = 0; sets[i]->array[j].retry = 0; sets[i]->array[j].status = F_okay; + sets[i]->array[j].write_state.conversion = f_conversion_data_base_10_c; + sets[i]->array[j].write_state.cache = &sets[i]->array[j].cache; if (main->program.parameters.arguments.array[index].used) { if (f_path_is_absolute(main->program.parameters.arguments.array[index]) == F_true || f_path_is_relative_current(main->program.parameters.arguments.array[index]) == F_true) { diff --git a/sources/c/tacocat/main/common/define.h b/sources/c/tacocat/main/common/define.h index 004a842..390047d 100644 --- a/sources/c/tacocat/main/common/define.h +++ b/sources/c/tacocat/main/common/define.h @@ -41,8 +41,10 @@ extern "C" { * - maintain: The max size in bytes to maintain a particular buffer. * * kt_tacocat_packet_*_d: - * - peek: The size to peek into the packet to get the initial information. - * - read: The size to read at a time when processing the packet. + * - minimum: The minimum packet size, "header:\npayload:\n" = 17. + * - peek: The size to peek into the packet to get the initial information. + * - prebuffer: A size used to include during allocation to add additional space for the packet header and therefore help reduce the potential number of allocations. + * - read: The size to read at a time when processing the packet. * * kt_tacocat_signal_*_d: * - check: When not using threads, this is how often to perform the check (lower numbers incur more kernel I/O). @@ -70,8 +72,10 @@ extern "C" { #define kt_tacocat_max_buffer_d 0x10000000 // 0x10^0x5 * 0x100 (Which is 256 Megabytes (0x10^0x5 where the base unit is 16 rather than 10 or 2 (maybe call this xytes? Megaxytes?)). #define kt_tacocat_max_maintain_d 0x100000 // 0x10^5 (Which is 1 Megabyte in base 16 (1 Megaxyte (MX)). - #define kt_tacocat_packet_peek_d F_fss_simple_packet_block_header_size_d - #define kt_tacocat_packet_read_d 0x2000 + #define kt_tacocat_packet_minimum_d 17 + #define kt_tacocat_packet_peek_d 64 + #define kt_tacocat_packet_prebuffer_d 0x200 + #define kt_tacocat_packet_read_d 0x2000 #define kt_tacocat_signal_check_d 0x4e20 #define kt_tacocat_signal_check_failsafe_d 0x4e20 diff --git a/sources/c/tacocat/main/common/enumeration.h b/sources/c/tacocat/main/common/enumeration.h index 2de34ec..e6ad73f 100644 --- a/sources/c/tacocat/main/common/enumeration.h +++ b/sources/c/tacocat/main/common/enumeration.h @@ -146,21 +146,25 @@ extern "C" { * Individual socket-specific flags for receiving. * * kt_tacocat_socket_flag_send_*_e: - * - none: No flags set. - * - file_open: File is open. - * - file_buffer: File is buffered. - * - connect: The remote address is connected. - * - buffer: Send the buffer. - * - buffer_sent: Buffer has been sent. + * - none: No flags set. + * - size: Determine the file size. + * - file: Buffer the file. + * - build: Build the header information. + * - connect: Connect to the remote address. + * - header: Send the header information. + * - payload: Send the payload information. + * - done: The entire Packet is sent. */ #ifndef _di_kt_tacocat_socket_flag_send_e_ enum { - kt_tacocat_socket_flag_send_none_e = 0x0, - kt_tacocat_socket_flag_send_file_open_e = 0x1, - kt_tacocat_socket_flag_send_file_buffer_e = 0x2, - kt_tacocat_socket_flag_send_connect_e = 0x4, - kt_tacocat_socket_flag_send_buffer_e = 0x8, - kt_tacocat_socket_flag_send_buffer_sent_e = 0x10, + kt_tacocat_socket_flag_send_none_e = 0x0, + kt_tacocat_socket_flag_send_size_e = 0x1, + kt_tacocat_socket_flag_send_file_e = 0x2, + kt_tacocat_socket_flag_send_build_e = 0x4, + kt_tacocat_socket_flag_send_connect_e = 0x8, + kt_tacocat_socket_flag_send_header_e = 0x10, + kt_tacocat_socket_flag_send_payload_e = 0x20, + kt_tacocat_socket_flag_send_done_e = 0x40, }; // enum #endif // _di_kt_tacocat_socket_flag_send_e_ diff --git a/sources/c/tacocat/main/common/print.h b/sources/c/tacocat/main/common/print.h index d664306..47098a6 100644 --- a/sources/c/tacocat/main/common/print.h +++ b/sources/c/tacocat/main/common/print.h @@ -39,14 +39,14 @@ extern "C" { */ #ifndef _di_kt_tacocat_f_e_ enum { - kt_tacocat_f_f_console_parameter_process_e, + kt_tacocat_f_f_console_parameter_process_e = 0, kt_tacocat_f_f_file_close_e, kt_tacocat_f_f_file_close_id_e, kt_tacocat_f_f_file_open_e, kt_tacocat_f_f_file_read_block_e, kt_tacocat_f_f_file_size_by_id_e, kt_tacocat_f_f_file_write_e, - kt_tacocat_f_f_fss_simple_packet_extract_range_e, + kt_tacocat_f_f_fss_payload_extract_range_e, kt_tacocat_f_f_memory_array_increase_by_e, kt_tacocat_f_f_memory_array_resize_e, kt_tacocat_f_f_network_from_ip_name_e, diff --git a/sources/c/tacocat/main/common/type.c b/sources/c/tacocat/main/common/type.c index f395f0c..a985eb7 100644 --- a/sources/c/tacocat/main/common/type.c +++ b/sources/c/tacocat/main/common/type.c @@ -54,10 +54,18 @@ extern "C" { f_memory_array_resize(0, sizeof(f_char_t), (void **) &array[i].buffer.string, &array[i].buffer.used, &array[i].buffer.size); } + if (array[i].cache.size) { + f_memory_array_resize(0, sizeof(f_char_t), (void **) &array[i].cache.string, &array[i].cache.used, &array[i].cache.size); + } + if (array[i].client.size) { f_memory_array_resize(0, sizeof(f_char_t), (void **) &array[i].client.string, &array[i].client.used, &array[i].client.size); } + if (array[i].header.size) { + f_memory_array_resize(0, sizeof(f_char_t), (void **) &array[i].header.string, &array[i].header.used, &array[i].header.size); + } + if (array[i].name.size) { f_memory_array_resize(0, sizeof(f_char_t), (void **) &array[i].name.string, &array[i].name.used, &array[i].name.size); } @@ -65,6 +73,14 @@ extern "C" { if (array[i].network.size) { f_memory_array_resize(0, sizeof(f_char_t), (void **) &array[i].network.string, &array[i].network.used, &array[i].network.size); } + + if (array[i].abstruses.size) { + f_memory_arrays_resize(0, sizeof(f_abstruse_map_t), (void **) &array[i].abstruses.array, &array[i].abstruses.used, &array[i].abstruses.size, &f_abstruse_maps_delete_callback); + } + + if (array[i].headers.size) { + f_memory_arrays_resize(0, sizeof(f_string_map_t), (void **) &array[i].headers.array, &array[i].headers.used, &array[i].headers.size, &f_string_maps_delete_callback); + } } // for } diff --git a/sources/c/tacocat/main/common/type.h b/sources/c/tacocat/main/common/type.h index e42fdc3..253fbc9 100644 --- a/sources/c/tacocat/main/common/type.h +++ b/sources/c/tacocat/main/common/type.h @@ -28,16 +28,23 @@ extern "C" { * socket: Socket structure. * status: The status of the socket operations. * - * buffer: A buffer for sending or receiving data between clients. - * client: A single client address for some network connection. - * name: A file name for reading from or writing to. - * network: A network name for the socket. - * packet: The simple packet range representing the parts of the packet for use during processing. + * buffer: A buffer for sending or receiving data between clients. + * cache: A cache used for various purposes, but primarily for the building of the send packet. + * client: A single client address for some network connection. + * header: A buffer used for the header part and possibly the reset of a packet, except for the actual payload. + * name: A file name for reading from or writing to. + * network: A network name for the socket. + * packet: The simple packet range representing the parts of the packet for use during processing. + * write_state: State data for the packet write operation. + * + * abstruses: The abstruse header map array, used for building the headers when sending a response. + * headers: The map of the headers after being processed for use when buulding the headers when sending a response. */ #ifndef _di_kt_tacocat_socket_set_t_ typedef struct { f_number_unsigned_t size_block; size_t size_done; + size_t size_total; uint16_t flag; uint16_t retry; @@ -46,10 +53,16 @@ extern "C" { f_status_t status; f_string_dynamic_t buffer; + f_string_dynamic_t cache; f_string_dynamic_t client; + f_string_dynamic_t header; f_string_dynamic_t name; f_string_dynamic_t network; f_fss_simple_packet_range_t packet; + f_fss_payload_header_write_state_t write_state; + + f_abstruse_maps_t abstruses; + f_string_maps_t headers; } kt_tacocat_socket_set_t; #define kt_tacocat_socket_set_t_initialize \ @@ -58,6 +71,7 @@ extern "C" { 0, \ 0, \ 0, \ + 0, \ f_file_t_initialize, \ f_socket_t_initialize, \ f_status_t_initialize, \ @@ -65,7 +79,12 @@ extern "C" { f_string_dynamic_t_initialize, \ f_string_dynamic_t_initialize, \ f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ f_fss_simple_packet_range_t_initialize, \ + f_fss_payload_header_write_state_t_initialize, \ + f_abstruse_maps_t_initialize, \ + f_string_maps_t_initialize, \ } #define macro_kt_tacocat_socket_set_t_initialize_1(size_block) \ @@ -74,6 +93,7 @@ extern "C" { 0, \ 0, \ 0, \ + 0, \ f_file_t_initialize, \ f_poll_t_initialize, \ f_socket_t_initialize, \ @@ -82,7 +102,35 @@ extern "C" { f_string_dynamic_t_initialize, \ f_string_dynamic_t_initialize, \ f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_fss_simple_packet_range_t_initialize, \ + f_fss_payload_header_write_state_t_initialize, \ + f_abstruse_maps_t_initialize, \ + f_string_maps_t_initialize, \ + } + + #define macro_kt_tacocat_socket_set_t_initialize_2(size_block, write_state) \ + { \ + size_block, \ + 0, \ + 0, \ + 0, \ + 0, \ + f_file_t_initialize, \ + f_poll_t_initialize, \ + f_socket_t_initialize, \ + f_status_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ + f_string_dynamic_t_initialize, \ f_fss_simple_packet_range_t_initialize, \ + write_state, \ + f_abstruse_maps_t_initialize, \ + f_string_maps_t_initialize, \ } #endif // _di_kt_tacocat_socket_set_t_ diff --git a/sources/c/tacocat/main/receive.c b/sources/c/tacocat/main/receive.c index 24b4f04..10a138c 100644 --- a/sources/c/tacocat/main/receive.c +++ b/sources/c/tacocat/main/receive.c @@ -222,10 +222,10 @@ extern "C" { return; } - set->status = f_fss_simple_packet_extract_range(set->buffer, &set->packet); - macro_kt_receive_process_begin_handle_error_exit_1(main, f_fss_simple_packet_extract_range, set->network, set->status, set->flag); + set->status = f_fss_payload_extract_range(set->buffer, &set->packet); + macro_kt_receive_process_begin_handle_error_exit_1(main, f_fss_payload_extract_range, set->network, set->status, set->flag); - if (set->status == F_packet_too_small || set->packet.size < F_fss_simple_packet_block_header_size_d) { + if (set->status == F_packet_too_small || set->packet.size < kt_tacocat_packet_minimum_d) { kt_tacocat_print_error_on_packet_too_small(&main->program.error, kt_tacocat_receive_s, set->network, kt_tacocat_packet_peek_d, set->buffer.used); set->buffer.used = 0; diff --git a/sources/c/tacocat/main/send.c b/sources/c/tacocat/main/send.c index 12cb840..eea7da2 100644 --- a/sources/c/tacocat/main/send.c +++ b/sources/c/tacocat/main/send.c @@ -79,33 +79,93 @@ extern "C" { return F_done_not; } - set->flag = kt_tacocat_socket_flag_send_file_open_e; + set->flag = kt_tacocat_socket_flag_send_size_e; } - if (set->flag == kt_tacocat_socket_flag_send_file_open_e) { + if (set->flag == kt_tacocat_socket_flag_send_size_e) { + + // Total is used here to explicitly pass a pointer of off_t rather than a pointer of size_t cast to an off_t. + off_t total = 0; + + set->status = f_file_size_by_id(set->file, &total); + + if (F_status_is_error(set->status)) { + kt_tacocat_print_error_on_file(&main->program.error, macro_kt_tacocat_f(f_file_size_by_id), kt_tacocat_send_s, set->network, set->status, set->name, f_file_operation_open_s); + + return F_done_not; + } + + set->size_total = (size_t) total; // @todo there is a size_total, but there is a total packet size and a total payload size. Find a way to distinguish both. + set->flag = kt_tacocat_socket_flag_send_file_e; + } + + if (set->flag == kt_tacocat_socket_flag_send_file_e) { set->buffer.used = 0; // Make sure the buffer is large enough for the file block reads (including a space for a terminating NULL). - set->status = f_memory_array_increase_by(set->file.size_read + 1, sizeof(f_char_t), (void **) &set->buffer.string, &set->buffer.used, &set->buffer.size); + set->status = f_memory_array_increase_by(set->file.size_read + f_fss_payload_object_payload_s.used + f_fss_payload_object_end_s.used + 1, sizeof(f_char_t), (void **) &set->buffer.string, &set->buffer.used, &set->buffer.size); macro_kt_send_process_handle_error_exit_1(main, f_memory_array_increase_by, set->network, set->status, set->flag); + set->status = f_string_dynamic_append(f_fss_payload_object_payload_s, &set->buffer); + macro_kt_send_process_handle_error_exit_1(main, f_file_read_block, set->network, set->status, set->flag); + + set->status = f_string_dynamic_append(f_fss_payload_object_end_s, &set->buffer); + macro_kt_send_process_handle_error_exit_1(main, f_file_read_block, set->network, set->status, set->flag); + set->status = f_file_read_block(set->file, &set->buffer); macro_kt_send_process_handle_error_exit_1(main, f_file_read_block, set->network, set->status, set->flag); - if (set->buffer.used) { - set->status = f_string_dynamic_terminate_after(&set->buffer); - macro_kt_send_process_handle_error_exit_1(main, f_string_dynamic_terminate_after, set->network, set->status, set->flag); + set->buffer.string[set->buffer.used] = 0; + set->flag = kt_tacocat_socket_flag_send_build_e; + } + + // @todo The build process needs to be broken up into multiple parts that can fit in a FSS Packet (2^32 - 6). + if (set->flag == kt_tacocat_socket_flag_send_build_e) { + set->header.used = 0; - set->flag = kt_tacocat_socket_flag_send_file_buffer_e; + if (set->buffer.used) { + set->status = f_memory_array_increase_by(kt_tacocat_packet_prebuffer_d, sizeof(f_char_t), (void **) &set->header.string, &set->header.used, &set->header.size); + macro_kt_send_process_handle_error_exit_1(main, f_memory_array_increase_by, set->network, set->status, set->flag); + + f_state_t state_local = main->setting.state; + state_local.data = &set->write_state; + + // @todo build the header and add to the header buffer and also add the "\npayload:\n". (resume programming here) + // The entire structure: + // f_fss_payload_comment_header_begin_s.used + f_fss_payload_comment_header_s.used + f_fss_payload_comment_header_end_s.used + // f_fss_payload_object_header_s.used + f_fss_payload_object_end_s + // set->header.used + // f_fss_payload_object_payload_s.used + f_fss_payload_object_end_s + // set->buffer.used + + // @todo create the following headers: + // 1) type file (or string, and if file, then name is filesize and size is filesize, otherwise name is unused and size is string length.) + // 1) name "example.csv" + // 2) size 1234 + // 3) status F_okay + + // @todo + // 1) Build abstruse headers map of data. + // 2) Call this: fl_fss_payload_header_map(set->headers, f_string_maps_t * const destinations, f_state_t * const state) + // 3) append maps to buffer. + + // @todo needs to be kt_tacocat_socket_flag_send_build_e equivalent for sending the structures, like the header structure. + + set->header.string[set->header.used] = 0; + set->flag = kt_tacocat_socket_flag_send_build_e; } else { // File is empty, nothing left to send. - set->flag = kt_tacocat_socket_flag_send_buffer_sent_e; + set->flag = kt_tacocat_socket_flag_send_done_e; } } - if (set->flag == kt_tacocat_socket_flag_send_file_buffer_e) { + // @todo all below is not updated to flag enumeration changes. + + // @todo don't send just yet, need to build the payload header (then send that first, followed by the payload itself).. + // probably should send the header data separately first to avoid additional allocations of merging the buffer with the header. + if (set->flag == kt_tacocat_socket_flag_send_build_e) { set->status = f_socket_connect(set->socket); macro_kt_send_process_handle_error_exit_1(main, f_socket_connect, set->network, set->status, set->flag); @@ -122,11 +182,11 @@ extern "C" { // When the buffer is smaller than the read block size, then the entire file should be completely sent. if (set->size_done == set->buffer.used && set->size_done < set->file.size_read) { - set->flag = kt_tacocat_socket_flag_send_buffer_sent_e; + set->flag = kt_tacocat_socket_flag_send_done_e; } } - if (set->flag == kt_tacocat_socket_flag_send_buffer_sent_e) { + if (set->flag == kt_tacocat_socket_flag_send_done_e) { set->flag = 0; set->size_done = 0; diff --git a/sources/c/tacocat/main/send.h b/sources/c/tacocat/main/send.h index 39408e8..a243933 100644 --- a/sources/c/tacocat/main/send.h +++ b/sources/c/tacocat/main/send.h @@ -34,6 +34,9 @@ extern "C" { /** * Process the network socket, retrieving the data and writing to the file. * + * This sends an FSS-000F (Simple Packet) using FSS-000E (Payload) such that the "payload" Object contains the file being transferred. + * The file being transferred is transferred as-is. + * * @param main * The main program and settings data. * diff --git a/sources/c/tacocat/main/tacocat.h b/sources/c/tacocat/main/tacocat.h index 8bd461e..b0b5dd7 100644 --- a/sources/c/tacocat/main/tacocat.h +++ b/sources/c/tacocat/main/tacocat.h @@ -40,6 +40,8 @@ // FLL-1 includes. #include +#include +#include #include // FLL-2 includes. -- 1.8.3.1