ERS packets time sensing filter and missing ones insertion

* Refactored echo meta * Additional fields parsed and packet debug * Add missing packets based on data record no. * Version/release document improved. * Calibration shortcoming note for release. * Executables for E2E and release are now explicit CMAKE_BUILD_TYPE=Release * Switch to RAM disk for E2E tests. * Deal with the ownership after running in docker. Stashing back is a separate step.
cgi-estonia-space · Nov 3, 2023 · 4e61011 · 4e61011
1 parent b0bd8c8
commit 4e61011
Show file tree

Hide file tree

Showing 13 changed files with 441 additions and 154 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -30,7 +30,6 @@ jobs:
           ip_address=$(cat trigger_out.txt | tail -n 1)
           echo "instance_ip=$ip_address" >> $GITHUB_OUTPUT
           sleep 5
-
   end-to-end-testing:
     name: End-to-end testing
     runs-on: ubuntu-latest
@@ -57,7 +56,9 @@ jobs:
           instance_ip=${{ needs.start-gpu-instance.outputs.instance_ip }}
           rm -rf $GITHUB_WORKSPACE/.git
           scp -o "StrictHostKeyChecking no" -r $GITHUB_WORKSPACE $INSTANCE_USERNAME@$instance_ip:/tmp/
-          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "CUDAARCHS=\"50;52\" /tmp/asar-focus/build-automation/create_exe_in_container.sh /tmp/asar-focus cgialus/alus-ootpa-devel-extra"
+          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "CUDAARCHS=\"50;52\" CMAKE_BUILD_TYPE=Release /tmp/asar-focus/build-automation/create_exe_in_container.sh /tmp/asar-focus cgialus/alus-ootpa-devel-extra"
+          # To RAM disk
+          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "cp -r /home/$INSTANCE_USERNAME/e2e /dev/shm/"
       - uses: actions/checkout@v4
       - name: End-to-end tests
         env:
@@ -73,12 +74,38 @@ jobs:
           echo "aws_access_key_id=${ACCESS_KEY_ID}" >> /tmp/cred
           echo "aws_secret_access_key=${SECRET_ACCESS_KEY}" >> /tmp/cred
           scp -o "StrictHostKeyChecking no" /tmp/cred $INSTANCE_USERNAME@$instance_ip:/tmp/
-          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "/tmp/asar-focus/build-automation/run_e2e_container.sh /home/$INSTANCE_USERNAME/e2e $exe_image_name /tmp/cred"
+          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "/tmp/asar-focus/build-automation/run_e2e_container.sh /dev/shm/e2e $exe_image_name /tmp/cred"
+  # Separate step because when tests fail we would like to analyze the results still
+  stash-assets-from-ramdisk:
+    name: Stash E2E assets from ramdisk to persistent disk
+    runs-on: ubuntu-latest
+    needs: [start-gpu-instance, end-to-end-testing]
+    if: |
+      needs.end-to-end-testing.result != 'skipped' &&
+      needs.end-to-end-testing.result != 'cancelled'
+    steps:
+      - name: Create SSH key
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.EAGLE_SSH_KEY }}
+        run: |
+          key_location=~/.ssh/
+          mkdir -p $key_location
+          key_path=$key_location/id_rsa
+          echo "$SSH_PRIVATE_KEY" > $key_path
+          sudo chmod 600 $key_path
+          ssh-keygen -f $key_path -y > $key_path.pub
+      - name: Modify ownership and move assets
+        env:
+          INSTANCE_USERNAME: ${{ secrets.EAGLE_USERNAME }}
+        run: |
+          instance_ip=${{ needs.start-gpu-instance.outputs.instance_ip }}
+          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "sudo chown -R $INSTANCE_USERNAME:$INSTANCE_USERNAME /dev/shm/e2e"
+          ssh -o "StrictHostKeyChecking no" $INSTANCE_USERNAME@$instance_ip "rsync -av /dev/shm/e2e /home/$INSTANCE_USERNAME/"
 
   stop-gpu-instance:
     name: Stop GPU instance
     runs-on: ubuntu-latest
-    needs: [start-gpu-instance, end-to-end-testing]
+    needs: [start-gpu-instance, end-to-end-testing, stash-assets-from-ramdisk]
     if: |
       always() &&
       needs.start-gpu-instance.result == 'success'

diff --git a/.github/workflows/release_package.yml b/.github/workflows/release_package.yml
@@ -27,7 +27,7 @@ jobs:
           CI_PACKAGE_FILENAME: ${{ env.CI_PACKAGE_NAME }}.tar.gz
           CI_PACKAGE_FILENAME_PATH: /tmp/${{ env.CI_PACKAGE_NAME }}.tar.gz
         run: |
-          cd $GITHUB_WORKSPACE/build-automation && CUDAARCHS="50;60;70;75;80" ./build_in_containers.sh $GITHUB_WORKSPACE /tmp/ cgialus/alus-ootpa-devel:latest cgialus/alus-devel:latest
+          cd $GITHUB_WORKSPACE/build-automation && CUDAARCHS="50;60;70;75;80" CMAKE_BUILD_TYPE=Release ./build_in_containers.sh $GITHUB_WORKSPACE /tmp/ cgialus/alus-ootpa-devel:latest cgialus/alus-devel:latest
           $GITHUB_WORKSPACE/build-automation/create_release_from_builds.sh /tmp/
           echo "package_path=/tmp/*_release.tar.gz" >> $GITHUB_ENV
           echo "package_filename=$CI_PACKAGE_FILENAME" >> $GITHUB_OUTPUT

diff --git a/RELEASE.md b/RELEASE.md
@@ -1,18 +1,50 @@
-# Release 0.2.0
+# Release 0.1.2
 
 ## Breaking changes
-* CLI arguments are defined and shall be used correctly in order to invoke processing
+* CLI arguments are defined and shall be used correctly in order to invoke processing, no more positional arguments
 
 ## Known Caveats
-* 
+* Only IMS product processing
+* Not all the auxiliary files are used/supported, only instrument (INS) and configuration (CON)
+* Range and azimuth compression windowing is yet to be done - https://github.com/cgi-estonia-space/asar-focus/issues/2
+* Processing speed (Vr) and Doppler centroid changes in azimuth direction yet to be done - https://github.com/cgi-estonia-space/asar-focus/issues/2
+  * This means that the focussing quality is not exactly on par with the reference processor
+* Packets' ISP sensing time handling might not be 100% correct
+  * It is observed for the reference products that it calculates new ISP sensing times based on PRI
+  * Therefore products by this processor differ in sensing start/stop and first/last line times (always inside the specified sensing filter)
+  * Best knowledge/effort basis changes has been implemented - https://github.com/cgi-estonia-space/asar-focus/issues/17 and https://github.com/cgi-estonia-space/asar-focus/issues/16
+* ERS time is not corrected according to PATM/N/C files - https://github.com/cgi-estonia-space/asar-focus/issues/15
+* Metadata for SQ_ADS, CHIRP_PARAM_ADS, etc. is not constructed currently - https://github.com/cgi-estonia-space/asar-focus/issues/10
+  * Other non DSD specific metadata as well - LEAP_SIGN, LEAP_ERR, ORBIT etc...
+* Final results' calibration constant is yet to be determined, currently it is not matching exactly the reference processor
+  * With the current experience/knowledge it must be experimented and tested, no known specific formula for it
 
 ## Major Features and Improvements
-*
+* Sensing start and end arguments
+* ERS and ENVISAT missing packets insertion
+* For ENVISAT all calibration packets are inserted with echo packet
+* Different modes/products generation is tracked and handled accordingly
+* Less explicit exits, everything is routed through exceptions with descriptive messages
+* Proper log system setup using Boost log, with 5 levels. Can be set when invoking the processor
+* Aux folder handling more robust - can identify aux files implicitly
+* Can parse proper orbit file out of the folder with miscellaneous aux files (TDS sets)
+* Echo metadata is ERS and ENVISAT specific, more fields parsed and used during processing (packet counters for example)
+* Diagnostic metric by default enabled, they are printed out after parsing step (different counters out of sequence and gap metrics)
+* Minor metadata improvements regarding processor, ref doc, compression method
+* FEP annotations' MJD for ENVISAT is parsed correctly
+* Faster parsing/preparing of echos from input dataset, from 1++ second to ~0.7 second
 
 ## Bug Fixes and Other Changes
+* Output file `FILE` handle check and other undefined states now guarded
+* Half of the modules are now CMake modules with unit tests
+* Can be compiled with clang (version 14 tested) as well, compiler flags and build improvements executed for this
 
 ## Thanks to our Contributors
 
+Kajal Haria, Fabiano Costantini from Telespazio UK\
+Sabrina Pinori, Marco Galli from SERCO\
+Andrea Recchia from aresys
+
 # Release 0.1.1
 
 ## Breaking changes
@@ -47,5 +79,6 @@
 
 ## Thanks to our Contributors
 
-Kajal Haria, Sabrina Pinori, Fabiano Costantini, Marco Galli from SERCO\
-Andrea Recchia from aresys
+Kajal Haria, Fabiano Costantini from Telespazio UK\
+Sabrina Pinori, Marco Galli from SERCO\
+Andrea Recchia from aresys
diff --git a/build-automation/compile_build.sh b/build-automation/compile_build.sh
@@ -14,7 +14,11 @@ fi
 repo_dir="$1"
 build_output="$2"
 
-cmake -B "${build_output}" "${repo_dir}"
+if [ -n "$CMAKE_BUILD_TYPE" ]; then
+  cmake_build_type_option="-DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE"
+fi
+
+cmake "${cmake_build_type_option}" -B "${build_output}" "${repo_dir}"
 cd "${build_output}" && make -j8
 
 result=0
@@ -30,4 +34,4 @@ if [ -n "$ALUS_ENABLE_TESTS" ]; then
 
 fi
 
-exit $result
+exit $result
diff --git a/build-automation/compile_build_container.sh b/build-automation/compile_build_container.sh
@@ -58,8 +58,12 @@ if [ -n "$ALUS_ENABLE_TESTS" ]; then
   tests_enabling_option="ALUS_ENABLE_TESTS=$ALUS_ENABLE_TESTS"
 fi
 
+if [ -n "$CMAKE_BUILD_TYPE" ]; then
+  cmake_build_type_option="CMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE"
+fi
+
 set +e
-docker exec -t "${container_name}" bash -c "$cuda_arch_value $tests_enabling_option ${container_work_dir_repo}/build-automation/compile_build.sh ${container_work_dir_repo} ${container_work_dir}/${image_name}"
+docker exec -t "${container_name}" bash -c "$cuda_arch_value $tests_enabling_option $cmake_build_type_option ${container_work_dir_repo}/build-automation/compile_build.sh ${container_work_dir_repo} ${container_work_dir}/${image_name}"
 status=$?
 docker stop "${container_name}"
 docker rm "${container_name}"

diff --git a/build-automation/create_exe_in_container.sh b/build-automation/create_exe_in_container.sh
@@ -45,10 +45,16 @@ docker run -t -d --name "${container_name}" "${docker_image}"
 docker cp "${repo_dir}" "${container_name}":"${container_work_dir}/"
 container_work_dir_repo="${container_work_dir}/${repo_folder_name}"
 echo "container_work_dir_repo - ${container_work_dir_repo}"
+
 if [ -n "$CUDAARCHS" ]; then
   cuda_arch_value="CUDAARCHS=\"$CUDAARCHS\""
 fi
-docker exec -t "${container_name}" bash -c "$cuda_arch_value ${container_work_dir_repo}/build-automation/compile_build.sh ${container_work_dir_repo} ${container_work_dir_repo}/build"
+
+if [ -n "$CMAKE_BUILD_TYPE" ]; then
+  cmake_build_type_option="CMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE"
+fi
+
+docker exec -t "${container_name}" bash -c "$cuda_arch_value $cmake_build_type_option ${container_work_dir_repo}/build-automation/compile_build.sh ${container_work_dir_repo} ${container_work_dir_repo}/build"
 docker exec -t "${container_name}" bash -c "ln -s ${container_work_dir_repo}/build/asar_focus /usr/bin/asar_focus"
 docker stop "${container_name}"
 docker commit "${container_name}" "${container_name}-exe"

diff --git a/envisat_format/include/envisat_lvl0_parser.h b/envisat_format/include/envisat_lvl0_parser.h
@@ -45,6 +45,7 @@ struct ASARMetadata {
     boost::posix_time::ptime sensing_stop;
     boost::posix_time::ptime first_line_time;
     boost::posix_time::ptime last_line_time;
+    bool product_err{false};
 
     std::string swath;
     std::string polarization;

diff --git a/envisat_format/include/envisat_types.h b/envisat_format/include/envisat_types.h
@@ -32,7 +32,11 @@ struct mjd {
     ul seconds;
     ul micros;
 
-    bool operator <(const mjd& o) const {
+    bool operator ==(const mjd& o) const{
+        return days == o.days && seconds == o.seconds && micros == o.micros;
+    }
+
+    bool operator <(const mjd& o) const{
         if (days == o.days) {
             if (seconds == o.seconds) {
                 return micros < o.micros;

diff --git a/envisat_format/include/ers_env_format.h b/envisat_format/include/ers_env_format.h
@@ -0,0 +1,33 @@
+/**
+ * ENVISAT and ERS ASAR instrument focusser for QA4EO activity (c) by CGI Estonia AS
+ *
+ * ENVISAT and ERS ASAR instrument focusser for QA4EO activity is licensed under a
+ * Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * You should have received a copy of the license along with this
+ * work. If not, see http://creativecommons.org/licenses/by-sa/4.0/
+ */
+#pragma once
+
+#include <cstddef>
+
+namespace alus::asar::envformat::ers {
+
+namespace highrate {
+// PX-SP-50-9105 3/1
+// https://earth.esa.int/eogateway/documents/20142/37627/ERS-products-specification-with-Envisat-format.pdf
+constexpr size_t MDSR_SIZE_BYTES{11498};
+constexpr size_t PROCESSOR_ANNOTATION_ISP_SIZE_BYTES{12};
+constexpr size_t FEP_ANNOTATION_SIZE_BYTES{20};
+constexpr size_t MDSR_ISP_SIZE_BYTES{11466};  // Adding ISP sensing (12) and FEP annotations (20) one gets sum above.
+constexpr size_t DATA_RECORD_NUMBER_SIZE_BYTES{4};
+constexpr size_t IDHT_HEADER_SIZE_BYTES{10};
+constexpr size_t AUXILIARY_REPLICA_CALIBRATION_SIZE_BYTES{220};
+constexpr size_t MEASUREMENT_DATA_SIZE_BYTES{11232};
+static_assert(DATA_RECORD_NUMBER_SIZE_BYTES + IDHT_HEADER_SIZE_BYTES + AUXILIARY_REPLICA_CALIBRATION_SIZE_BYTES +
+                  MEASUREMENT_DATA_SIZE_BYTES ==
+              MDSR_ISP_SIZE_BYTES);
+
+}  // namespace highrate
+
+}  // namespace alus::asar::envformat::ers
diff --git a/envisat_format/src/envisat_im_parse.cc b/envisat_format/src/envisat_im_parse.cc
@@ -294,6 +294,7 @@ void ParseEnvisatLevel0ImPackets(const std::vector<char>& file_data, const DSD_l
         FEPAnnotations fep;
         it = CopyBSwapPOD(fep, it);
 
+        // This is actually application process ID plus some bits, needs to be refactored, more info - PO-ID-DOR-SY-0032
         uint16_t packet_id;
         it = CopyBSwapPOD(packet_id, it);
 

diff --git a/envisat_format/src/envisat_lvl1_writer.cc b/envisat_format/src/envisat_lvl1_writer.cc
@@ -245,9 +245,12 @@ void WriteLvl1(const SARMetadata& sar_meta, const ASARMetadata& asar_meta, MDS&
 
         mph.Set_PRODUCT(out_name);
         mph.Set_PROC_STAGE('N');
-        // https://earth.esa.int/eogateway/documents/20142/37627/PO-RS-507316_4_C_Envisat_Product_Spec_Vol8.pdf/
         if (asar_meta.product_name.substr(0, 3) == "ASA") {
+            // https://earth.esa.int/eogateway/documents/20142/37627/PO-RS-507316_4_C_Envisat_Product_Spec_Vol8.pdf/
             mph.Set_REF_DOC("PO-RS-MDA-GS-2009_4/C");
+        } else if (asar_meta.product_name.substr(0, 3) == "SAR") {
+            // https://earth.esa.int/eogateway/documents/20142/37627/ERS-products-specification-with-Envisat-format.pdf
+            mph.Set_REF_DOC("PX-SP-50-9105_3/1");
         } else {
             mph.Set_REF_DOC("TBD.pdf");
         }
@@ -263,7 +266,7 @@ void WriteLvl1(const SARMetadata& sar_meta, const ASARMetadata& asar_meta, MDS&
         mph.SetOrbitInfo(asar_meta);
         mph.Set_SBT_Defaults();
         mph.Set_LEAP_Defaults();
-        mph.Set_PRODUCT_ERR('0');
+        mph.Set_PRODUCT_ERR(asar_meta.product_err ? '1' : '0');
 
         // set tot size later