0.8.2 yaml pod.manifest, read site config first

- yaml (configuration & metadata)
  site config,
  pod:
    pod.manifest,
    doc header:
      make,
      meta
- pod.manifest in yaml breaks
  - older program reading new pod and
  - newer program reading old pod
- read config first,
  - get site default language in case it is absent
    in pod.manifest
  -  ensure config is not read too many times
     (it is at present, revisit)

5 files changed, 208 insertions, 48 deletions

diff --git a/org/default_paths.org b/org/default_paths.org
index 6fa9078..d122c9a 100644
--- a/org/default_paths.org
+++ b/org/default_paths.org
@@ -49,7 +49,7 @@ template PodManifest() {
   mixin DocReformRgxInit;
   static auto rgx = Rgx();
   auto PodManifest(P)(
-    P   _pth
+    P   _pth=""
   ) {
     struct ManifestFile_ {
       string pod_manifest_filename() {
@@ -424,9 +424,8 @@ template PathMatters() {
               if (!exists(_output_path)) {
                 try {
                   _output_path.mkdirRecurse;
-                }
-                // catch (ErrnoException ex) {
-                catch (Exception ex) {
+                // } catch (ErrnoException ex) {
+                } catch (Exception ex) {
                   // Handle error
                 }
               }
@@ -697,6 +696,10 @@ template DocReformPathsPods() {
           pth ~ " == "  ~ _base_dir_pod ~ "/" ~ base_filename_(fn_src) ~ "?");
         return pth;
       }
+      string output_pod_manifest_file(string fn_src) {
+        string pth = base_filesystem_(fn_src).chainPath("pod.manifest").array;
+        return pth;
+      }
       string base_pod_(string fn_src) {
         string pth = _base_dir_pod.chainPath(base_filename_(fn_src)).array; // change this
         return pth;
@@ -741,8 +744,8 @@ template DocReformPathsPods() {
         return _pods();
       }
       auto pod_root(string fn_src) {
-        auto pth_1_ = "";
-        auto pth_2_ = ((base(fn_src).filesystem_open_zpod.chainPath("")).asNormalizedPath).array;
+        auto pth_1_ = "pod";
+        auto pth_2_ = ((base(fn_src).filesystem_open_zpod.chainPath("")).asNormalizedPath).array; // "dr_doc"
         struct _pods {
           auto zpod() {
             return pth_1_;
@@ -783,9 +786,24 @@ template DocReformPathsPods() {
         }
         return _pods();
       }
+      auto pod_manifest(string fn_src) {
+        auto pod_root_ = pod_root(fn_src);
+        auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("pod.manifest")).asNormalizedPath).array;
+        auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("pod.manifest")).asNormalizedPath).array;
+        struct _pods {
+          auto zpod() {
+            return pth_1_;
+          }
+          auto filesystem_open_zpod() {
+            assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_);
+            return pth_2_;
+          }
+        }
+        return _pods();
+      }
       auto media_root(string fn_src) {
         auto pod_root_ = pod_root(fn_src);
-        auto pth_1_ = "media";
+        auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("media")).asNormalizedPath).array;
         auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("media")).asNormalizedPath).array;
         struct _pods {
           auto zpod() {
diff --git a/org/doc_reform.org b/org/doc_reform.org
index abba098..6b08fd2 100644
--- a/org/doc_reform.org
+++ b/org/doc_reform.org
@@ -28,7 +28,7 @@ struct Version {
   int minor;
   int patch;
 }
-enum _ver = Version(0, 8, 1);
+enum _ver = Version(0, 8, 2);
 #+END_SRC
 
 ** compilation restrictions (supported compilers)
@@ -807,7 +807,16 @@ auto _env = [
 #+BEGIN_SRC d
 auto _manifest_matter = PathMatters!()(_opt_action, _env, "");
 auto _manifests = [ _manifest_matter ];
+ConfCompositePlus _make_and_meta_struct_tst;
 foreach(arg; args[1..$]) {
+  _manifest_matter = PathMatters!()(_opt_action, _env, arg); // gather input markup source file names for processing
+  auto _config_local_site_struct = readConfigSite!()(_manifest_matter, _env);
+  { /+ local site config +/
+    if (_config_local_site_struct.filetype == "yaml") {
+      import doc_reform.meta.conf_make_meta_yaml;
+      _make_and_meta_struct_tst = _config_local_site_struct.configParseYAMLreturnDocReformStruct!()(_make_and_meta_struct_tst, _manifest_matter);
+    }
+  }
   auto _manifest_start = PodManifest!()(arg);
   if (arg.match(rgx.flag_action)) {
     flag_action ~= " " ~ arg;   // flags not taken by getopt
@@ -827,21 +836,104 @@ foreach(arg; args[1..$]) {
     if (exists(sisudoc_txt_)) {
       try {
         if (exists(sisudoc_txt_)) {
-          pod_manifest_root_content_paths_to_markup_location_raw_ = sisudoc_txt_.readText;
+          import dyaml;
+          try {
+            Node pod_manifest_yaml = Loader.fromFile(sisudoc_txt_).load();
+            if ("doc" in pod_manifest_yaml) {
+              if (pod_manifest_yaml["doc"].type.mapping
+                && pod_manifest_yaml["doc"].tag.match(rgx.yaml_tag_is_map)
+              ) {
+                if ("path" in pod_manifest_yaml["doc"]) {
+                  if (pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_seq)) {
+                    foreach (string _path; pod_manifest_yaml["doc"]["path"]) {
+                      markup_contents_location_ ~= _path ~ "\n";
+                      pod_manifest_root_content_paths_to_markup_location_raw_ ~=
+                        _path ~ "\n";
+                    }
+                  } else if (
+                    pod_manifest_yaml["doc"]["path"].type.string
+                    && pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_str)
+                  ) {
+                    markup_contents_location_ = pod_manifest_yaml["doc"]["path"].get!string;
+                    pod_manifest_root_content_paths_to_markup_location_raw_ =
+                      pod_manifest_yaml["doc"]["path"].get!string;
+                  }
+                }
+                if ("filename" in pod_manifest_yaml["doc"]) {
+                  if (pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_seq)) {
+                    foreach (string _filename; pod_manifest_yaml["doc"]["filename"]) {
+                      if ("language" in pod_manifest_yaml["doc"]) {
+                        if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) {
+                          foreach (string _lang; pod_manifest_yaml["doc"]["language"]) {
+                            markup_contents_location_ ~=
+                              "media/text/"
+                              ~ _lang ~ "/"
+                              ~ _filename ~ "\n";
+                          }
+                        } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str)
+                        ) {
+                          markup_contents_location_ =
+                            "media/text/"
+                            ~ pod_manifest_yaml["doc"]["language"].get!string
+                            ~ "/" ~ _filename ~ "\n";
+                        } else {
+                          string _lang_default = "en";
+                          markup_contents_location_ ~=
+                            "media/text/"
+                            ~ _lang_default ~ "/"
+                            ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                        }
+                      } else {
+                        string _lang_default = "en";
+                        markup_contents_location_ ~=
+                          "media/text/"
+                          ~ _lang_default ~ "/"
+                          ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                      }
+                    }
+                  } else if (
+                    pod_manifest_yaml["doc"]["filename"].type.string
+                    && pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_str)
+                  ) {
+                    if ("language" in pod_manifest_yaml["doc"]) {
+                      if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) {
+                        foreach (string _lang; pod_manifest_yaml["doc"]["language"]) {
+                          markup_contents_location_ ~=
+                            "media/text/"
+                            ~ _lang ~ "/"
+                            ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                        }
+                      } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str)) {
+                        markup_contents_location_ =
+                          "media/text/"
+                          ~ pod_manifest_yaml["doc"]["language"].get!string
+                          ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                      } else {
+                        string _lang_default = "en";
+                        markup_contents_location_ ~=
+                          "media/text/"
+                          ~ _lang_default ~ "/"
+                          ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                      }
+                    } else {
+                      string _lang_default = "en";
+                      markup_contents_location_ ~=
+                        "media/text/"
+                        ~ _lang_default ~ "/"
+                        ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n";
+                    }
+                  }
+                }
+              }
+            }
+
+          } catch (ErrnoException ex) {
+          }
         }
       } catch (ErrnoException ex) {
       } catch (FileException ex) {
         // Handle errors
       }
-      if (pod_manifest_root_content_paths_to_markup_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+
-        foreach (m; pod_manifest_root_content_paths_to_markup_location_raw_.matchAll(rgx.pod_content_location)) {
-          foreach (n; m.captures[2].matchAll(rgx.language_codes)) {
-            markup_contents_location_ ~= "media/text/" ~ n.captures[1].to!string ~ "/" ~ m.captures[1].to!string ~ "\n";
-          }
-        }
-      } else { // (file name with path \n)+
-        markup_contents_location_ = pod_manifest_root_content_paths_to_markup_location_raw_;
-      }
     } else {
       writeln("manifest not found: ", sisudoc_txt_);
     }
diff --git a/org/output_hub.org b/org/output_hub.org
index 5375b2f..e3b4607 100644
--- a/org/output_hub.org
+++ b/org/output_hub.org
@@ -82,11 +82,23 @@ import doc_reform.output,
 
 #+name: output_scheduled_task
 #+BEGIN_SRC d
-if (sched == outTask.pod) {
-  msg.v("doc reform source processing... ");
+if (sched == outTask.source
+  || sched == outTask.pod
+) {
+  if (sched == outTask.source) {
+    msg.v("doc reform source processing... ");
+  }
+  if (sched == outTask.pod) {
+    msg.v("doc reform source pod processing... ");
+  }
   import doc_reform.output.source_pod;
   DocReformPod!()(doc_matters);
-  msg.vv("doc reform source done");
+  if (sched == outTask.source) {
+    msg.vv("doc reform source done");
+  }
+  if (sched == outTask.pod) {
+    msg.vv("doc reform source pod done");
+  }
 }
 #+END_SRC
 
diff --git a/org/output_src_pod.org b/org/output_src_pod.org
index 1fd6f6c..1e61c72 100644
--- a/org/output_src_pod.org
+++ b/org/output_src_pod.org
@@ -31,15 +31,15 @@ template DocReformPod() {
       <<source_pod_mkdirs>>
       <<source_pod_copy>>
       <<source_pod_zip>>
-    }
-    catch (ErrnoException ex) {
+    } catch (ErrnoException ex) {
       // Handle error
     }
   }
 }
 #+END_SRC
 
-** imports
+** init
+*** imports
 
 #+name: output_imports
 #+BEGIN_SRC d
@@ -55,7 +55,7 @@ import
   doc_reform.output.xmls;
 #+END_SRC
 
-** mkdir                                                             :mkdir:
+*** init
 
 #+name: source_pod_init
 #+BEGIN_SRC d
@@ -73,6 +73,8 @@ static auto rgx = Rgx();
 assert (doc_matters.src.filename.match(rgx.src_fn));
 #+END_SRC
 
+** mkdir                                                             :mkdir:
+
 #+name: source_pod_mkdirs
 #+BEGIN_SRC d
 /+ create directory structure +/
@@ -148,7 +150,8 @@ auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
         zip.addMember(zip_arc_member_file);
       }
     } else {
-      if (doc_matters.opt.action.verbose) {
+      if (doc_matters.opt.action.debug_do
+      && doc_matters.opt.action.verbose) {
         writeln("WARNING (io) src out NOT found (image): ", fn_src_in);
       }
     }
@@ -179,19 +182,19 @@ auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
       zip.addMember(zip_arc_member_file);
     }
   } else {
-    if (doc_matters.opt.action.verbose
-    || doc_matters.opt.action.debug_do) {
+    if (doc_matters.opt.action.debug_do
+    && doc_matters.opt.action.verbose) {
       writeln("WARNING (io) src out NOT found (document make): ", fn_src_in);
     }
   }
-} { /+ bundle primary file +/
+} { /+ pod manifest +/
   auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
   auto fn_src_out_pod_zip_base
-    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string;
+    = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string;
   auto fn_src_out_filesystem
-    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // without root path:
+    = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path
   auto fn_src_out_inside_pod
-    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // without root path:
+    = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path
   string[] filelist_src_out_pod_arr;
   string[] filelist_src_zpod_arr;
   if (exists(fn_src_in)) {
@@ -200,14 +203,51 @@ auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
     }
     filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
     filelist_src_zpod_arr ~= fn_src_out_inside_pod;
-    if (doc_matters.opt.action.source) {
-      auto filelist
+    {
+      import dyaml;
+      auto pod_filelist_yaml_string
         = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w");
-      foreach (source_pth_and_fn; filelist_src_zpod_arr) {
-        filelist.writeln(source_pth_and_fn);
+      Node _pmy;
+      string _pm = "doc:\n  filename: " ~ doc_matters.src.filename ~ "\n  language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n";
+      if (doc_matters.opt.action.debug_do
+      && doc_matters.opt.action.verbose) {
+        _pmy = Loader.fromString(_pm).load();
+        writeln("pod filename:  ", _pmy["doc"]["filename"].get!string);
+        writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string);
+        writeln("pod languages: ", doc_matters.src.language);
+        // foreach(string _l; _pmy["doc"]["language"]) {
+        //   writeln("language:      ", _l);
+        // }
+      }
+      if (doc_matters.opt.action.source) {
+        pod_filelist_yaml_string.writeln(_pm);
+      }
+      if (doc_matters.opt.action.pod) {
+        auto zip_arc_member_file = new ArchiveMember();
+        zip_arc_member_file.name = fn_src_out_pod_zip_base;
+        auto zip_data = new OutBuffer();
+        zip_data.write(_pm);
+        zip_arc_member_file.expandedData = zip_data.toBytes();
+        zip.addMember(zip_arc_member_file);
       }
-      fn_src_in.copy(fn_src_out_filesystem);
     }
+  }
+} { /+ bundle primary file (.ssm/.sst) +/
+  auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string;
+  auto fn_src_out_pod_zip_base
+    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string;
+  auto fn_src_out_filesystem
+    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path:
+  auto fn_src_out_inside_pod
+    = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path:
+  string[] filelist_src_out_pod_arr;
+  string[] filelist_src_zpod_arr;
+  if (exists(fn_src_in)) {
+    debug(io) {
+      writeln("(io debug) src in found: ", fn_src_in);
+    }
+    filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base;
+    filelist_src_zpod_arr ~= fn_src_out_inside_pod;
     if (doc_matters.opt.action.pod) {
       auto zip_arc_member_file = new ArchiveMember();
       zip_arc_member_file.name = fn_src_out_pod_zip_base;
@@ -217,12 +257,12 @@ auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
       zip.addMember(zip_arc_member_file);
     }
   } else {
-    if (doc_matters.opt.action.verbose
-    || doc_matters.opt.action.debug_do) {
+    if (doc_matters.opt.action.debug_do
+    && doc_matters.opt.action.verbose) {
       writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in);
     }
   }
-} { /+ bundle insert files +/
+} { /+ bundle insert files (.ssi) +/
   if (doc_matters.srcs.file_insert_list.length > 0) {
     foreach (insert_file; doc_matters.srcs.file_insert_list) {
       debug(pod) {
@@ -265,8 +305,8 @@ auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
           createZipFile!()(fn_pod, zip.build());
         }
       } else {
-        if (doc_matters.opt.action.verbose
-        || doc_matters.opt.action.debug_do) {
+        if (doc_matters.opt.action.debug_do
+        && doc_matters.opt.action.verbose) {
           writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in);
         }
       }
@@ -293,7 +333,8 @@ auto fn_pod = pths_pod.pod_filename(doc_matters.src.filename).zpod;
 #+BEGIN_SRC d
 if (exists(fn_pod)) {
   try {
-    if (!(doc_matters.opt.action.quiet)) {
+    if (!(doc_matters.opt.action.quiet)
+    && doc_matters.opt.action.pod) {
       auto data = (cast(byte[]) (fn_pod).read);
       if (doc_matters.opt.action.verbose) {
         writeln(" ", doc_matters.src.filename, " > ");
@@ -307,8 +348,7 @@ if (exists(fn_pod)) {
           auto data = zipped.expand(member);
           writeln("> ", filename, " length ", data.length);
         }
-      }
-      catch (ZipException ex) {
+      } catch (ZipException ex) {
         // Handle errors
       }
       if (doc_matters.src.filename == "dr_doc/media/text/en/the_wealth_of_networks.yochai_benkler.sst") {
@@ -332,8 +372,7 @@ if (exists(fn_pod)) {
         );
       }
     }
-  }
-  catch (ErrnoException ex) {
+  } catch (ErrnoException ex) {
     // Handle errors
   }
 }
diff --git a/org/source_files_read.org b/org/source_files_read.org
index ecf6cc7..ebc6a4d 100644
--- a/org/source_files_read.org
+++ b/org/source_files_read.org
@@ -193,7 +193,6 @@ static template configReadSiteYAML() {
 }
 static template configReadDocYAML() {
   <<imports_std>>
-  import yaml;
   final YAMLDocument configReadDocYAML(M,E)(M _manifest, E _env) {
     string _configuration = configReadInDocYAML!()(_manifest, _env);
     auto _conf_file_details = ConfigFilePaths!()(_manifest, _env);