checkint before trying to switch over to datafiles

README.md

@@ -5,12 +5,11 @@ to generate multi-lingual curricula documentation tables from
 structured representations as a flatfile database. Data scheme and 
 actual values are kept in YAML files in order to version them with git.
 
-# Usage
+## Usage
 
 ```sh
-$> python coursebuilder  
-usage:  [-h] [-m META [META ...]] [-l LANG] [-f FIELDS [FIELDS ...]] [-s SCHEMA] [-p] [-t] [-b BOOK] [--level LEVEL]
-        [--table-gen TABLE_GEN]
+usage:  [-h] [-m META [META ...]] [-l LANG] [-f FIELDS [FIELDS ...]] [-s SCHEMA] [-q QUERY] [-qs QUERY_SORT] [-qc QUERY_COMPOUND] [-qf QUERY_FILTER [QUERY_FILTER ...]]
+        [-p] [--title TITLE] [-b BOOK] [--level LEVEL] [--table-gen TABLE_GEN] [--template TEMPLATE] [-o OUT] [--legacy] [--leftcol LEFTCOL]
 
 versatile curricula generator
 
@@ -23,19 +22,31 @@ options:
                         Fields to be used, the table will be build accordingly
   -s SCHEMA, --schema SCHEMA
                         using provided schema
+  -q QUERY, --query QUERY
+                        compound query to select items
+  -qs QUERY_SORT, --query-sort QUERY_SORT
+                        sort query with a min/max over a column like min:credits
+  -qc QUERY_COMPOUND, --query-compound QUERY_COMPOUND
+                        create a compound from a column with multiple values/dictionaries in cells
+  -qf QUERY_FILTER [QUERY_FILTER ...], --query-filter QUERY_FILTER [QUERY_FILTER ...]
+                        filter final list of columns for output
   -p, --pagebreak       add a pagebreak after each module
-  -t, --title           take first value in list as title
+  --title TITLE         template for title - use curly brackets (i.e. {}) to mark where the title string is inserted
   -b BOOK, --book BOOK  process a whole curriculum book with sections
   --level LEVEL         level of header tags
   --table-gen TABLE_GEN
                         runs table generator
+  --template TEMPLATE   defines a template to be used with fields
+  -o OUT, --out OUT     set the output type
+  --legacy              use legacy generator mode for compatibility
+  --leftcol LEFTCOL     maximum size of left column
 ```
 
-# Author
+## Author
 
 © Copyright 2020-2024 Hartmut Seichter
 
-# Licence
+## Licence
 
 Coursebuilder is licensed under the terms of the MIT License. For details consult https://opensource.org/license/mit/ or the attached license file
 

TODO.md

@@ -6,4 +6,5 @@
 * [x] add a book mode for mixing input and headers (# Blah -m mod.cg.yaml)
 * [~] table generator
 * [ ] overlay of compulsory with other modes ... 
-* [ ] add template based generator
+* [ ] add template based generator
+* [ ] port over to structured YAML ... https://tolgee.io/platform/formats/structured_yaml

coursebuilder/__main__.py

@@ -1,64 +1,186 @@
 #!/usr/bin/env python
 
 """
-CourseBuilder 
+CourseBuilder
 
-Coursebuilder is a preprocessor tool for [pandoc](https://pandoc.org) 
-to generate multi-lingual curricula documentation tables from 
-structured representations as a flatfile database. Data scheme and 
+Coursebuilder is a preprocessor tool for [pandoc](https://pandoc.org)
+to generate multi-lingual curricula documentation tables from
+structured representations as a flatfile database. Data scheme and
 actual values are kept in YAML files in order to version them with git.
 
 """
 
 from argparse import ArgumentParser
-import yaml
-import string
 import os,sys
+import yaml
+import pandas as pd
+from string import Template
 
 from tablegenerator import TableGenerator
 from markdowngenerator import MarkdownGenerator
 from templategenerator import TemplateGenerator
-from metagenerator import MetaGenerator
+from schema import Schema
 
 
 class CourseBuilder:
 
+    @staticmethod
+    def generate(args):
+        if args.schema and args.meta:
+
+            # get actual fields
+            actual_fields = None
+
+            # use a file instead of list
+            if args.fields and os.path.isfile(args.fields[0]):
+                with open(args.fields[0]) as ff:
+                    actual_fields = yaml.load(ff,Loader=yaml.Loader)['fields']
+            else:
+                # seem we have a list or None
+                actual_fields = args.fields
+
+            # get schema
+            schema = None
+            with open(args.schema) as f:
+                schema = Schema(yaml.load(f,Loader=yaml.Loader))
+
+            # if no fields are given, take all!
+            if actual_fields == None:
+                actual_fields = list(schema.keys())
+
+
+            result_df = []
+
+            # iterate through meta files
+            for m in args.meta:
+                with open(m) as fm:
+
+                    if args.legacy:
+
+                        MarkdownGenerator.generate_table_legacy(
+                            table_items=schema.to_list_of_tuple(
+                                meta=yaml.load(fm,Loader=yaml.Loader),
+                                fields=actual_fields,
+                                lang=args.lang),
+                            add_pagebreak=args.pagebreak,
+                            title_template=args.title,
+                            first_colwidth=args.leftcol)
+                    elif args.query:
+
+                        lot = schema.to_short_dict(
+                                meta=yaml.load(fm,Loader=yaml.Loader),
+                                fields=actual_fields,
+                                lang=args.lang)
+
+                        result_df.append(pd.DataFrame([lot]))
+                    else:
+                            MarkdownGenerator.generate_table(
+                            table_items=schema.to_list_of_tuple(
+                                meta=yaml.load(fm,Loader=yaml.Loader),
+                                fields=actual_fields,
+                                lang=args.lang),
+                            add_pagebreak=args.pagebreak,
+                            title_template=args.title,
+                            first_colwidth=args.leftcol)
+
+            # query mode
+            if args.query and len(result_df):
+
+                # got the list
+                df = pd.concat(result_df,ignore_index=True)
+
+                # generate a dataframe
+                df_q = df.query(args.query)
+
+                # generate a compound column --query-compound column:sum
+                if args.query_compound:
+                    # print('{}.sum'.format(args.query_compound))
+                    df_q.loc[:,'{}.sum'.format(args.query_compound)] = df_q[args.query_compound].apply(lambda x: sum(list(x.values())))
+                    print(df_q)
+
+                # --query-sort is parameterized as min:credits - hence direction:column
+                if args.query_sort:
+                    qs = args.query_sort.split(':')
+                    match qs[0]:
+                        case 'min' : df_q = df_q.sort_values(by=qs[1],ascending=True,key=lambda col: min(col) if hasattr(col,'__len()__') else col)
+                        case 'max' : df_q = df_q.sort_values(by=qs[1],ascending=False,key=lambda col: max(col) if hasattr(col,'__len()__') else col)
+
+                # filter query
+                if args.query_filter:
+                    df_q = df_q.loc[:,args.query_filter]
+
+                # print(df_q.head())
+
+                # set value transforms
+                if args.query_template:
+
+                    # no idea yet how to parameterize this
+                    ww = 'written'
+                    #df_q['form-of-exam'] = 'Schriftlich' if df_q.loc[:,'form-of-exam'] == 'written' else 'was anderes'
+                    # mm = Template("{'written':'S','oral':'mündlich'}[${v}]")?
+                    # print(mm.format(v=mm))
+
+                # lets get crazy to create a summary table!
+                # df_summary = pd.DataFrame([{
+                #     'sum.credits': df_q['credits'].sum()
+                # }])
+
+                # set labels directly!
+                if args.query_labels:
+                    df_q.columns = args.query_labels
+
+                q_as_md = df_q.to_markdown(tablefmt='grid',index=False)
+
+                print(q_as_md)
+
+                # print(df_summary.to_markdown(tablefmt='grid',index=False))
+
+
     @staticmethod
     def run():
-        
+
         # arguments
         parser = ArgumentParser(description='versatile curricula generator')
 
+        # loading mode for internal database
         parser.add_argument('-m','--meta',action="extend", nargs="+", type=str,help="course description(s) as YAML file(s)")
         parser.add_argument('-l','--lang',help="Language to parse from meta file (use de or en)",default='de')
         parser.add_argument('-f','--fields',help="Fields to be used, the table will be build accordingly",action="extend", nargs="+", type=str)
-        parser.add_argument('-s','--schema',help="using provided schema")
+        parser.add_argument('-s','--schema', help="using provided schema")
+
+        # query mode
+        parser.add_argument('-q','--query', type=str, default=None, help="compound query to select items")
+        parser.add_argument('-qs','--query-sort',type=str,default=None,help="sort query with a min/max over a column like min:credits")
+        parser.add_argument('-qc','--query-compound',type=str,default=None,help="create a compound from a column with multiple values/dictionaries in cells")
+        parser.add_argument('-qf','--query-filter',type=str,default=[],action="extend", nargs="+",help="filter final list of columns for output")
+        parser.add_argument('-ql','--query-labels',type=str,default=[],action="extend", nargs="+",help="new labels for query like")
+        parser.add_argument('-qt','--query-template',type=str,default=[],action="extend", nargs="+",help="templates for values in the form of {value}")
+
+
+        # create pagebreaks
         parser.add_argument('-p','--pagebreak',action="store_true",help="add a pagebreak after each module")
-        parser.add_argument('-t','--title',action="store_true",help="take first value in list as title")
+        parser.add_argument('--title',type=str,default=None,help="template for title - use curly brackets (i.e. {}) to mark where the title string is inserted")
         parser.add_argument('-b','--book',type=str,help="process a whole curriculum book with sections")
         parser.add_argument('--level',type=int,default=1,help="level of header tags")
         parser.add_argument('--table-gen',type=str,default=None,help='runs table generator')
         parser.add_argument('--template',type=str,default=None,help='defines a template to be used with fields')
-        
+        parser.add_argument('-o','--out',type=str,default=None,help='set the output type')
+        parser.add_argument('--legacy',action="store_true",help="use legacy generator mode for compatibility")
+
+        parser.add_argument('--leftcol',type=int,default=35,help='maximum size of left column')
+
         # get arguments
         args = parser.parse_args()
-        
+
         if args.table_gen:
 
             tg = TableGenerator()
-            
             tg.generate_table(args.table_gen)
 
             return
 
-
         # book mode with predefined setting from a book file
         if args.book and args.schema:
-            
-            generator = MetaGenerator()
-
-            with open(args.schema) as sf:
-                generator.set_schema(yaml.load(sf,Loader=yaml.Loader))
 
             with open(args.book) as bf:
 
@@ -68,63 +190,36 @@ class CourseBuilder:
                 book_path = os.path.abspath(args.book)
 
                 for bi in book['book']:
+
                     if 'fields' in bi:
                         actual_fields = bi['fields']
+
                     if 'sections' in bi:
                         for section in bi['sections']:
+
                             if 'text' in section:
                                 print(section['text'][args.lang])
+
+                            # gernerate section wise parts
                             if 'modules' in section:
-                                for m in section['modules']: 
-                                    mod_path = os.path.join(os.path.dirname(book_path),m)
 
-                                    with open(mod_path) as fm:
-                                        try:
+                                # override fields
+                                args.fields = actual_fields
 
-                                            table_items = generator.process(yaml.load(fm,Loader=yaml.Loader),fields=actual_fields,lang=args.lang,pagebreak=args.pagebreak,createTitle=args.title,header_level=args.level)
-                                            
-                                            MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level)
-                                            
-                                        except Exception as exc:
-                                            print(f'{type(exc).__name__} in {mod_path}: {exc}',file=sys.stderr)
+                                # expand filenames to be relative to the book
+                                args.meta = [os.path.join(os.path.dirname(book_path),mod_path) for mod_path in section['modules']]
 
+                                CourseBuilder.generate(args=args)
 
         # verbose command line mode
-        elif args.schema and args.meta and len(args.fields) > 0:
-            
-            # get actual fields
-            actual_fields = []
-
-            if os.path.isfile(args.fields[0]):
-                with open(args.fields[0]) as ff:
-                    actual_fields = yaml.load(ff,Loader=yaml.Loader)['fields']
-            else:
-                actual_fields = args.fields 
-            
-        
-            # get schema
-            actual_schema = None
-            with open(args.schema) as f:
-                actual_schema = yaml.load(f,Loader=yaml.Loader)
-
-            # iterate through meta files
-            for m in args.meta:
-                with open(m) as fm:
-        
-                    generator = MetaGenerator()
-                    generator.set_schema(actual_schema)
-
-                    table_items = generator.process(yaml.load(fm,Loader=yaml.Loader),fields=actual_fields,lang=args.lang,pagebreak=args.pagebreak,createTitle=args.title,header_level=args.level,template=args.template)
-
-                    if args.template:
-                        TemplateGenerator.generate(table_items)
-                    else:
-                        MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level)
-
-                    # print(table_items)
-
+        elif args.schema:
+            CourseBuilder.generate(args=args)
         else:
             parser.print_help()
 
+# run as main
 if __name__ == '__main__':
+    # recommended setting for pandas
+    pd.options.mode.copy_on_write = True
+    # run
     CourseBuilder.run()

coursebuilder/markdowngenerator.py

@@ -1,25 +1,44 @@
 #!/usr/bin/env python
 
-import textwrap,itertools
-
-# we need raw value maybe add a third item - tuple the input?
-
-# alternative use a dictionary
-# { name: XYZ }
-
-# or make it the class
+import itertools
 
 class MarkdownGenerator:
 
     @staticmethod
     def generate_tablerow() -> str:
         pass
+
+
+    @staticmethod
+    def generate_table(table_items,add_pagebreak = False,title_template = None,first_colwidth = 28):
+
+        import pandas as pd
+        import tabulate 
+
+        # get the dataframe
+        df = pd.DataFrame(table_items)
+
+        # use first column for 
+        df.columns = df.iloc[0]
+        df = df[1:]
+
+        if title_template != None:
+            print(title_template.format(df.columns[1]),'\n')
+
+        print(df.to_markdown(tablefmt='grid', index=False, maxcolwidths=[first_colwidth,None]))
+        print('\n') # always add a newline after the table
+
+        if add_pagebreak:
+            print('\\pagebreak')
+        
     
     @staticmethod
-    def generate(ti,pagebreak = False,title = False,header_level = 1) -> str:
+    def generate_table_legacy(table_items,add_pagebreak = False,title_template = None,first_colwidth = 28):
+
+        import textwrap
 
         line_length = 128
-        column_ratio= 0.28
+        column_ratio = float(first_colwidth) / 100
 
         h_len = int(line_length * column_ratio)
         d_len = line_length-h_len
@@ -27,8 +46,8 @@ class MarkdownGenerator:
         #
         #  generate title (currently the first one)
         # 
-        if title:
-            print('#' * header_level,ti[0][1],'\n')
+        if title_template != None:
+            print(title_template.format(table_items[0][1]),'\n')
 
         print(''.join(['+',"".ljust(h_len,'-'),'+',"".ljust(d_len,'-'),'+']))
 
@@ -40,7 +59,7 @@ class MarkdownGenerator:
 
         # test if this affected by a third item!
 
-        for k,v in ti:
+        for k,v in table_items:
 
             # 
             if v == None:
@@ -79,5 +98,5 @@ class MarkdownGenerator:
 
 
         # to control pagebreaks for pandoc
-        if pagebreak:
+        if add_pagebreak:
             print('\n\\newpage')

coursebuilder/metagenerator.py

@@ -1,155 +0,0 @@
-import os,string,sys
-import yaml
-
-class MetaGenerator:
-
-    def __init__(self) -> None:
-        self.__schema = None
-
-    def set_schema(self,schema = None):
-        self.__schema = schema
-
-    def get_template(self,field,lang='de'):
-        if 'template' in self.__schema[field]:
-            return self.__schema[field]['template'][lang]
-        else:
-            return "$value"        
-        
-    def is_translatable(self,field):
-        if 'translatable' in self.__schema[field]:
-            return self.__schema[field]['translatable']
-        else:
-            return True        
-
-    def needs_spec(self,field):
-        if 'spec' in self.__schema[field]:
-            return self.__schema[field]
-        else:
-            return False
-        
-    def process_label(self,field,lang='de'):
-        # processes the label of a field item
-        return self.__schema[field]['label'][lang]
-
-    def process_str(self,meta,field,lang='de'):
-        if self.is_translatable(field):
-
-            return [self.process_label(field,lang),meta[field][lang]]
-        else:
-            if not 'value' in meta[field]:
-                raise AssertionError(field,'incomplete')
-            
-            return [self.process_label(field,lang),meta[field]['value']]
-
-    def process_enum(self,meta,field,lang='de'):
-        """
-        enum have a specification 'specs' option 
-        that can be forced by the scheme
-        """
-        vv = meta[field]['value']
-        enum_val = self.__schema[field]['values'][vv][lang]
-
-        if self.needs_spec(field):
-
-            t = string.Template(self.get_template(field=field,lang=lang))
-
-            spec = meta[field]['spec'][lang]
-
-            return [self.process_label(field,lang),t.substitute({'value': enum_val,'spec': spec})]
-        else:
-            return [self.process_label(field,lang),enum_val]
-
-    def process_num(self,meta,field,lang='de'):
-        v = meta[field]['value']
-        t = string.Template(self.get_template(field,lang))
-        return [self.process_label(field,lang),t.substitute({'value' : v})]
-    
-    def process_multinum(self,meta,field,lang='de'):
-        v = meta[field]['value']
-        t = string.Template(self.get_template(field,lang))
-        if hasattr(v, "__len__"):
-            vv = [t.substitute({'value' : ev}) for ev in v]
-            return [self.process_label(field,lang),', '.join(vv)]
-        else:
-            return self.process_num(meta=meta,field=field,lang=lang)
-    
-    
-    def process_multikey(self,meta,field,lang='de'):
-        """
-        multikey need to assign a numeric value to a key
-        """
-        vs = meta[field]['value']
-        t = string.Template(self.get_template(field,lang))
-
-        k = self.process_label(field,lang)
-
-        parts = []
-
-        for e in vs:
-            kk = self.__schema[field]['keys'][e][lang]
-            parts.append(t.substitute({'key': kk, 'value' : vs[e]}))
-
-        return [k,', '.join(parts)]
-
-
-    def process(self,meta,fields = [],lang = 'de',pagebreak = False,createTitle=False,header_level=1,template=None):
-
-        table_items = []
-
-        # iterate over requested fields
-        for field in fields:
-            try:
-                # correlate with schema and append
-                match self.__schema[field]['type']: 
-                    case 'str': table_items.append(self.process_str(meta,field,lang)) 
-                    case 'enum': table_items.append(self.process_enum(meta,field,lang))
-                    case 'int' | 'num' : table_items.append(self.process_num(meta,field,lang))
-                    case 'multinum' : table_items.append(self.process_multinum(meta,field,lang))
-                    case 'multikey': table_items.append(self.process_multikey(meta,field,lang))
-            except Exception as exp:
-                print(field,' not resolvable in ',self.__schema,exp)
-
-        # maybe return tableitems as np.Dataframe?
-        return table_items
-
-    
-
-        # if template != None:
-        #     # use template generator
-        #     TemplateGenerator.generate(table_items,pagebreak,createTitle,header_level=header_level)
-        #     pass
-        # else:
-        #     # conventional MD mode
-        #     MarkdownGenerator.generate(table_items,pagebreak,createTitle,header_level=header_level)
-
-    # def process_book_section(self,section,lang='de'):
-    #     pass
-
-    # book mode
-    # def process_book(self,book,bookpath,create_title,pagebreak,lang='de',header_level=2):
-
-    #     actual_fields = []
-
-    #     for bi in book['book']:
-    #         if 'fields' in bi:
-    #             actual_fields = bi['fields']
-    #         if 'sections' in bi:
-    #             for section in bi['sections']:
-    #                 if 'text' in section:
-    #                     print(section['text'][lang])
-    #                 if 'modules' in section:
-    #                     for m in section['modules']: 
-    #                         mod_path = os.path.join(os.path.dirname(bookpath),m)
-
-    #                         with open(mod_path) as fm:
-    #                             try:
-    #                                 table_items = self.process(yaml.load(fm,Loader=yaml.Loader),fields=actual_fields,lang=lang,pagebreak=pagebreak,createTitle=create_title,header_level=header_level)
-
-    #                                 print(table_items)
-                                    
-    #                             except Exception as exc:
-    #                                 print(f'{type(exc).__name__} in {mod_path}: {exc}',file=sys.stderr)
-
-
-
-

coursebuilder/schema.py

@@ -0,0 +1,87 @@
+import string
+
+class Schema:
+
+    def __init__(self,schema) -> None:
+        self.__schema = schema
+
+    def __getitem__(self, field):
+        return self.__schema[field]
+
+    def keys(self):
+        return self.__schema.keys()
+        
+    def is_translatable(self,field):
+        if 'translatable' in self.__schema[field]:
+            return self.__schema[field]['translatable']
+        else:
+            return True        
+
+    def needs_spec(self,field):
+        if 'spec' in self.__schema[field]:
+            return self.__schema[field]
+        else:
+            return False
+ 
+    def get_value(self,meta,field,lang):
+        """
+        treats receiving the value like a variant, 
+        returns values with their language specific representations
+        """
+        match self.__schema[field]['type']: 
+            case 'str': return meta[field][lang] if self.is_translatable(field) else meta[field]['value']
+            case 'enum' | 'int' | 'num' | 'multikey' : return meta[field]['value']
+            case 'multinum': return meta[field]['value'] if hasattr(meta[field]['value'],'__iter__') else [meta[field]['value'],] # force list!
+    
+    def to_list_of_dict(self,meta,fields,lang):
+        """
+        generates a list of dict which can easily be converted 
+        to a pandas dataframe
+        """
+        # list comprehension for rows
+        return [{'field' : field,   # field name
+                  'lang' : lang,    # language shortcode
+                  'type' : self.__schema[field]['type'], # datatype
+                  'label' : self.__schema[field]['label'][lang], # label
+                  'value' : self.get_value(meta,field,lang), # actual value
+                  'template' : self.__schema[field]['template'][lang] if 'template' in self.__schema[field] else None,
+                  # getting crazy with nested dict comprehension
+                  'enum_values' : { k:v[lang] for (k,v) in self.__schema[field]['values'].items()} if 'enum' in self.__schema[field]['type'] else None,
+                  'key_values' : { k:v[lang] for (k,v) in self.__schema[field]['keys'].items()} if 'multikey' in self.__schema[field]['type'] else None,
+                  'spec' : meta[field]['spec'][lang] if 'spec' in meta[field] else None
+                  }
+                  for field in fields]
+    
+
+    def to_short_dict(self,meta,fields,lang):
+        """
+        generates a short version of dict which can easily be converted 
+        to a pandas dataframe
+        """
+        # dict comprehension for whole meta part
+        return { field : self.get_value(meta,field,lang) for field in fields }
+    
+    def to_list_of_tuple(self,meta,fields,lang):
+        """
+        generates a list of tuples with a label and value (text)
+        this is usually consumed by a Markdown generator
+
+        todo: needs deuglyfication of free standing loop, templates are possible for all
+        """
+        list = []
+        for r in self.to_list_of_dict(meta,fields,lang):
+            match r['type']:
+                case 'str' : 
+                    list.append( (r['label'],r['value']) )
+                case 'int' | 'num' : 
+                    list.append( ( r['label'], r['template'].format(value=r['value'],spec=r['spec']) if r['template'] else r['value']) )
+                case 'enum' : 
+                    list.append( ( r['label'], r['template'].format(value=r['enum_values'][r['value']],spec=r['spec']) 
+                                  if r['template'] else r['enum_values'][r['value']] ) )
+                case 'multikey' : 
+                    list.append( ( r['label'], ', '.join( [r['template'].format(key=r['key_values'][k],value=v) for k,v in r['value'].items()] ) ) )
+                case 'multinum' :
+                    list.append( (r['label'], ', '.join( r['template'].format(value=v) for v in r['value'])) )
+
+        return list
+

coursebuilder/tablegenerator.py

@@ -4,6 +4,8 @@ import string
 import tempfile
 import subprocess
 import os
+import pandas as pd
+import tabulate
 
 class TableGenerator:
     """

docs/quickstart.md

@@ -0,0 +1,41 @@
+# concept
+
+The concept behind coursebuilder is to store curricula descriptions in `YAML` files that can be versioned in a git repository. Unlike classic databases, an observable and well defined versioning is paramount in these descriptions as they are the legal foundation for study and exam regulations.
+
+The following pieces play together here:
+
+- `schema` files, usually a `schema.yaml`
+- `mod` files, usually something along the lines of `mod.coursecode.yaml`
+- `book` files describing a whole regulation set and course global details
+- some sort of transformation with `coursebuilder` into Markdown that is piped through [pandoc](https://pandoc.org) in order to generate PDF, HTML and other representation from this code
+
+# schema files
+
+Schema files are responsible to describe the used structures in a database. The following datatypes are supported:
+
+- `str` a simple string, can be accompanied with a `template`
+- `enum` a classic enum datatype with a fixed set of values
+- `num` a numeric datatype
+- `multinum` an array type with the possibility to `spec` each value
+- `multikey` a key-value type with additional numeric data associated with each key instance
+
+# mod files (modules)
+
+Modules describe a course in detail and implement an instance of the schema file. Especially `strings` and `enums` are translatable One of the plan is to use a validator to find inconsistencies automatically, like workloads that are not following the 30h = 1ECTS rule.
+
+
+# datatypes
+
+## `str` datatype
+
+```yaml
+# this would reside in a schema field on top level
+# a field of name 'id'
+id: # name of the field
+    type: str # sets the datatype to str
+    translatable: false # enforces the value is not translatable (default is true)
+    label: { # label describes the meaning of the datatype in regards of the schema
+        de: "Kürzel",   # translation of the label in German (de)
+        en: "code" # translation of the label in English (en)
+    }
+```

requirements.txt

@@ -0,0 +1,7 @@
+numpy==1.26.4
+pandas==2.2.2
+python-dateutil==2.9.0.post0
+pytz==2024.1
+six==1.16.0
+tabulate==0.9.0
+tzdata==2024.1

test/Makefile

@@ -0,0 +1,44 @@
+# debug make file for testing
+build_dir := build
+target_en := ${build_dir}/table.en.pdf
+target_de := ${build_dir}/table.de.pdf
+target_de_book := ${build_dir}/curricullum.de.pdf
+
+targets := ${target_de} ${target_en} ${target_de_book}
+
+target_flags := --template pandoc-template/eisvogel.latex -V table-use-row-colors:true
+
+coursebuilder := ../coursebuilder
+
+all: ${targets}
+
+${target_en}: mod.cg.yaml
+	@echo "creating English version ..."
+	mkdir -p ${build_dir}
+	python ${coursebuilder} -s schema.yaml -m $^ -l en -f fields.yaml | pandoc ${target_flags} -o ${target_en}
+
+${target_de}: mod.cg.yaml
+	@echo "creating German version ..."
+	mkdir -p ${build_dir}
+	python ${coursebuilder} -s schema.yaml -m $^ -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de}
+
+${target_de_book}: *.yaml
+	python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 25 --legacy | pandoc ${target_flags} -V toc:true -V lang:de -o ${target_de_book}
+
+clean:
+	rm -f ${targets}
+
+debug:
+	python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --title "## {}" -l de -f name credits goal content
+	#  | pandoc ${target_flags} -V lang:de -o ${target_de}
+
+debug-query:
+	python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name credits form-of-exam -ql Modulname Kreditpunkte Prüfungsart -qt quatsch
+
+debug-query-book:
+	python ${coursebuilder} -s schema.yaml -b book.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name credits form-of-instruction -ql Modulname Kürzel Kreditpunkte
+
+debug-query-full:
+	python ${coursebuilder} -s ~/Documents/MaACS/MHB/schema.yaml -b ~/Documents/MaACS/MHB/book.yaml -q "kind=='compulsory_elective'" -qc form-of-instruction -qf name form-of-instruction.sum credits term -ql Modulname SWS Kreditpunkte Semester
+
+.PHONY: clean

test/book.yaml

@@ -4,8 +4,9 @@
 # 
 
 book:
-  - fields: 
-    - name 
+  - fields:
+    - name
+    - instructor 
     - id
     - goal
     - content
@@ -33,5 +34,21 @@ book:
         en: "## elective courses {.unnumbered}"
     - modules: 
         - mod.interactsys.yaml
+        - mod.test.yaml
 
 
+# 
+# tables
+#
+query:
+  list-credits-and-workload:
+    - fields:
+      - name
+      - credits
+      - workload
+
+# just for ideas
+regulations:
+  - globals:
+    - course_name: Applied Computer Science (M.Sc.)
+     

test/fields.yaml

@@ -1,5 +1,6 @@
 fields: 
-  - name 
+  - name
+  - instructor
   - id
   - goal
   - content

test/mod.cg.yaml

@@ -2,6 +2,9 @@ name:
   de: Computergrafik
   en: Computer Graphics
 
+instructor:
+  de: Prof. Hartmut Seichter, PhD
+  en: Prof. Hartmut Seichter, PhD
 
 id:
   value: CG

test/mod.test.yaml

@@ -0,0 +1,106 @@
+name:
+  de: Test Vorlesung
+  en: Lecture of Test
+
+instructor: 
+  de: Cicero
+  en: Cicero
+
+id:
+  value: Test
+
+credits: 
+  value: 5
+
+form-of-exam: 
+  value: written
+
+form-of-instruction:
+  value: { 'lecture': 2, 'exersise': 1 }
+
+term:
+  value: [1, 3]
+
+duration:
+  value: 1
+
+kind: 
+  value: compulsory
+
+goal: 
+  de: |
+      **What is it**
+
+      Lorem Ipsum is simply dummy text of the printing and typesetting 
+      industry. Lorem Ipsum has been the industry's standard dummy text 
+      ever since the 1500s, when an unknown printer took a galley of type 
+      and scrambled it to make a type specimen book. It has survived not only 
+      five centuries, but also the leap into electronic typesetting, remaining 
+      essentially unchanged. It was popularised in the 1960s with the release 
+      of Letraset sheets containing Lorem Ipsum passages, and more recently with
+      desktop publishing software like Aldus PageMaker including versions of 
+      Lorem Ipsum.
+
+
+
+  en: | 
+      
+
+content: 
+  de: |
+      **Where did it come from**
+
+      Contrary to popular belief, Lorem Ipsum is not simply random text. 
+      It has roots in a piece of classical Latin literature from 45 BC, 
+      making it over 2000 years old. Richard McClintock, a Latin professor 
+      at Hampden-Sydney College in Virginia, looked up one of the more 
+      obscure Latin words, consectetur, from a Lorem Ipsum passage, and 
+      going through the cites of the word in classical literature, 
+      discovered the undoubtable source. Lorem Ipsum comes from sections 
+      1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The 
+      Extremes of Good and Evil) by Cicero, written in 45 BC. This book 
+      is a treatise on the theory of ethics, very popular during the 
+      Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor 
+      sit amet..", comes from a line in section 1.10.32.
+       
+   
+  en: |
+     
+
+teaching-material:
+  de: |
+   
+  en: |
+    
+
+prerequisites:
+  de: ""
+  en: ""
+
+author-of-indenture: 
+  de: ""
+  en: ""
+
+used-in:
+  de: "Master Applied Computerscience"
+  en: "Master Applied Computerscience"
+
+workload:
+  de: "2SWS Vorlesung 1SWS Übung"
+  en: "2SWS lecture 1SWS exersise"
+
+form-of-exam:
+  value: written
+  spec: 
+    de: "120min Klausur"
+    en: "120min exam"
+  
+frequency:
+  value: once_per_year
+
+kind:
+  value: compulsory
+
+remarks:
+  de: 
+  en:

test/schema.yaml

@@ -0,0 +1,228 @@
+# fields in curricular description
+# leaning on methods in OpenAPI 3.0
+
+#
+# Modulname
+#
+name:
+  type: str
+  label:
+    de: "Modulname"
+    en: "name of course"
+
+#
+# Modulverantwortliche:r
+#
+instructor:
+  type: str
+  label:
+    de: "Modulverantwortlicher / Modulverantwortliche"
+    en: "module instructor"
+
+#
+# Kürzel / ID
+#
+id:
+  type: str
+  translatable: false
+  label: { de: "Kürzel", en: "code" }
+
+#
+# Qualifikationsziele
+#
+
+# Welche fachbezogenen, methodischen, fachübergreifende Kompetenzen,
+# Schlüsselqualifikationen - werden erzielt (erworben)? Diese sind
+# an der zu definierenden Gesamtqualifikation (angestrebter Abschluss) auszurichten.
+#
+# Lernergebnisse sind Aussagen darüber, was ein Studierender nach Abschluss des Moduls weiß,
+#  versteht und in der Lage ist zu tun. Die Formulierung sollte sich am Qualifikationsrahmen
+# für Deutsche Hochschulabschlüsse orientieren und Inhaltswiederholungen vermeiden.
+#
+# Des Weiteren finden Sie im QM-Portal die „Handreichung zur Beschreibung von Lernzielen“
+# als Formulierungshilfe.
+
+goal:
+  type: str
+  label: { de: "Qualifikationsziele", en: "educational goal" }
+
+#
+# Modulinhalte
+#
+
+# Welche fachlichen, methodischen, fachpraktischen und fächerübergreifenden
+# Inhalte sollen vermittelt werden?
+#
+# Es ist ein stichpunktartiges Inhaltsverzeichnis zu erstellen.
+
+content:
+  type: str
+  label: { de: "Modulinhalte", en: "content" }
+
+#
+# Lehrform
+#
+
+#
+# Welche Lehr- und Lernformen werden angewendet?
+# (Vorlesungen, Übungen, Seminare, Praktika,
+# Projektarbeit, Selbststudium)
+#
+# Es sind nur Werte aus der Prüfungsordung zugelassen
+#
+form-of-instruction:
+  type: multikey
+  label: { de: "Lehrform(en)", en: "form of instruction" }
+  keys:
+    {
+      "lecture": { de: "Vorlesung", en: "lecture" },
+      "lecture_seminar":
+        { de: "Seminaristische Vorlesung", en: "lecture and seminar" },
+      "seminar": { de: "Seminar", en: "seminar" },
+      "exersise": { de: "Übung", en: "lab exersise" },
+      "pc_lab": { de: "Rechnergestütztes Praktikum", en: "PC exersise" },
+      "project": { de: "Project", en: "project" },
+    }
+  template:
+    de: "{key} ({value}SWS)"
+    en: "{key} ({value}SWS)"
+
+#
+# Voraussetzungen für die Teilnahme
+#
+
+# Für jedes Modul sind die Voraussetzungen für die Teilnahme zu beschreiben.
+# Welche Kenntnisse, Fähigkeiten und Fertigkeiten sind für eine
+# erfolgreiche Teilnahme vorauszusetzen?
+#
+# Alternativ können die Module benannt werden welche für die erfolgreiche
+# Teilnahme im Vorfeld zu belegen sind.
+
+prerequisites:
+  type: str
+  label: { de: "Voraussetzungen für die Teilnahme", en: "prerequisites" }
+
+#
+# Literatur und multimediale Lehr- und Lernprogramme
+#
+#
+# Wie können die Studierenden sich auf die Teilnahme an diesem Modul vorbereiten?
+#
+teaching-material:
+  type: str
+  label:
+    {
+      de: "Literatur und multimediale Lehr- und Lernprogramme",
+      en: "media of instruction",
+    }
+
+#
+# Lehrbriefautor
+#
+author-of-indenture:
+  type: str
+  label: { de: "Lehrbriefautor", en: "author of indenture" }
+
+#
+# Verwendung in (Studienprogramm)
+#
+used-in:
+  type: str
+  label: { de: "Verwendung", en: "used in study programs" }
+
+#
+# Arbeitsaufwand
+#
+workload:
+  type: str
+  label: { de: "Arbeitsaufwand / Gesamtworkload", en: "workload" }
+#
+# credits/ECTS
+#
+credits:
+  type: num
+  unit: ECTS
+  label:
+    {
+      en: "credits and weight of mark",
+      de: "Kreditpunkte und Gewichtung der Note in der Gesamtnote",
+    }
+  template:
+    de: "{value}CP, Gewichtung: {value}CP von 120CP "
+    en: "{value}CP, weight: {value} / 120 "
+
+#
+# Leistungsnachweis
+#
+form-of-exam:
+  type: enum
+  label: { de: "Leistungsnachweis", en: "form of examination" }
+  values:
+    {
+      "written": { de: "Schriftliche Prüfung", en: "written exam" },
+      "oral": { de: "Mündliche Prüfung", en: "oral exam" },
+      "alternative":
+        { de: "Alternative Prüfungunsleistung", en: "alternative examination" },
+    }
+  spec: true
+  template:
+    de: "{value} ({spec})"
+    en: "{value} ({spec})"
+
+#
+# Semester
+#
+term:
+  type: multinum
+  label: { de: "Semester", en: "term" }
+  template:
+    de: "{value}\\. Semester"
+    en: "{value}\\. semester"
+
+#
+# Häufigkeit des Angebots
+#
+frequency:
+  type: enum
+  label: { de: "Häufigkeit des Angebots", en: "frequency of Offer" }
+  values:
+    {
+      "once_per_term": { de: "jedes Semester", en: "every semester" },
+      "once_per_year":
+        { de: "einmal im Studienjahr", en: "once per study year" },
+    }
+
+#
+# Dauer des Angebots
+#
+duration:
+  type: int
+  label:
+    de: Dauer
+    en: duration
+  template:
+    de: "{value} Semester"
+    en: "{value} term(s)"
+
+#
+# Art der Veranstaltung
+#
+kind:
+  type: enum
+  label:
+    {
+      de: "Art der Veranstaltung (Pflicht, Wahl, etc.)",
+      en: "kind of module (compulsory, elective)",
+    }
+  values:
+    {
+      "compulsory": { de: "Pflicht", en: "compulsory" },
+      "elective": { de: "Wahl/Wahlpflicht", en: "elective" },
+    }
+
+#
+# Freiform Bemerkungen
+#
+remarks:
+  type: str
+  label: { de: "Besonderes", en: "remarks" }

test/simple/Makefile

@@ -1,28 +0,0 @@
-
-coursebuilder := ../../coursebuilder
-
-table.en.pdf:
-	@echo "creating English version ..."
-	python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l en -f fields.yaml | pandoc --template pandoc-template/eisvogel.latex -o table.en.pdf
-
-table.de.pdf:
-	@echo "creating German version ..."
-	python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc --template pandoc-template/eisvogel.latex -o table.de.pdf
-
-all: table.en.pdf table.de.pdf
-
-clean: 
-	rm -f table.en.pdf table.de.pdf
-
-debug-template:
-	python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits --template "$$name | $$credits"
-
-debug-markdown:
-	python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits 
-
-debug-book:
-	python ${coursebuilder} -s schema.yaml -b book.yaml -l de 
-
-debug:
-	python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml
-

test/simple/mod.test.yaml

@@ -1,98 +0,0 @@
-
-name:
-  en: Test Course
-
-
-test:
-  
-  competency-table:
-    de:
-      - "Lineare Algebra": 'ABC'
-      - "Vector Spaces": 'A'
-
-
-
-#
-# nested lists seem to work in Markdown only in the US style way
-#
-# reference here: https://meta.stackexchange.com/questions/85474/how-to-write-nested-numbered-lists
-#
-# note the parser actually corrects 'Tervuren' to 3 in resulting data
-#
-
-content:
-  en: |
-    1. Blah
-
-    2. Blub
-
-        1. Blah
-
-        1. Blub
-
-        1. Blah
-
-        1. Blub
-
-        1. Blah
-
-        1. Blub
-
-        1. Blah
-
-        1. Blub
-
-    3. Blah
-
-    4. Blub
-
-    <!-- break -->
-
-    5. Blah
-
-    6. Blah and Blub
-
-        1. Blah
-
-        1. Blub
-
-    7. Blah and Blub
-
-      - Blah
-
-      - Blub
-
-      - Blah
-
-      - Blub
-
-    8. Blub and Blah
-
-      - Blah
-
-      - Blub
-
-      - Blah
-
-      - Blub
-
-      - Blah
-
-      - Blub
-
-      - Blah
-
-      - Blub
-
-    9. Blah, Blub and Blub
-
-      - Blah
-
-      - Blub
-
-      - Blah
-
-      - Blub
-
-
-

test/simple/schema.yaml

@@ -1,291 +0,0 @@
-# fields in curricular description
-# leaning on methods in OpenAPI 3.0 
-
-#
-# Modulname
-#
-name:
-    type: str
-    label: 
-        de: "Modulname"
-        en: "name of course"
-
-#
-# Modulverantwortliche:r
-#
-instructor: 
-    type: str
-    translatable: false
-    label: 
-        de: "Modulverantwortlicher/Modulverantwortliche"
-        en: "module instructor"
-    
-
-#
-# Kürzel / ID
-#
-id: 
-    type: str
-    translatable: false
-    label: {
-        de: "Kürzel",
-        en: "code"
-    }
-
-
-#
-# Qualifikationsziele
-#
-
-# Welche fachbezogenen, methodischen, fachübergreifende Kompetenzen, 
-# Schlüsselqualifikationen - werden erzielt (erworben)? Diese sind 
-# an der zu definierenden Gesamtqualifikation (angestrebter Abschluss) auszurichten.
-# 
-# Lernergebnisse sind Aussagen darüber, was ein Studierender nach Abschluss des Moduls weiß,
-#  versteht und in der Lage ist zu tun. Die Formulierung sollte sich am Qualifikationsrahmen 
-# für Deutsche Hochschulabschlüsse orientieren und Inhaltswiederholungen vermeiden.
-# 
-# Des Weiteren finden Sie im QM-Portal die „Handreichung zur Beschreibung von Lernzielen“ 
-# als Formulierungshilfe.
-
-goal:
-    type: str
-    label: {
-        de: "Qualifikationsziele",
-        en: "educational goal"
-    }
-
-#
-# Modulinhalte
-#
-
-# Welche fachlichen, methodischen, fachpraktischen und fächerübergreifenden 
-# Inhalte sollen vermittelt werden? 
-# 
-# Es ist ein stichpunktartiges Inhaltsverzeichnis zu erstellen.
-
-content: 
-    type: str
-    label: {
-        de: "Modulinhalte",
-        en: "content"
-    }
-
-#
-# Lehrform
-#
-
-#
-# Welche Lehr- und Lernformen werden angewendet? 
-# (Vorlesungen, Übungen, Seminare, Praktika, 
-# Projektarbeit, Selbststudium)
-#
-# Es sind nur Werte aus der Prüfungsordung zugelassen
-# 
-form-of-instruction: 
-    label: {
-        de: "Lehrform(en)",
-        en: "form of instruction"
-    }
-    type: multikey 
-    keys: {
-        'lecture' : {
-            de: "Vorlesung",
-            en: "lecture"
-        },
-        'lecture_seminar' : {
-            de: "Seminaristische Vorlesung",
-            en: "lecture and seminar"
-        },
-        'seminar' : {
-            de: "Seminar",
-            en: "seminar"
-        },
-        'exersise' : {
-            de: "Übung",
-            en: "lab exersise"
-        },
-        'pc_lab' : {
-            de: "Rechnergestütztes Praktikum",
-            en: "PC exersise"
-        },
-        'project' : {
-            de: "Project",
-            en: "project"
-        }
-    }
-    template: 
-        de: "${key} (${value}SWS)"
-        en: "${key} (${value}SWS)"
-
-#
-# Voraussetzungen für die Teilnahme
-#
-
-# Für jedes Modul sind die Voraussetzungen für die Teilnahme zu beschreiben. 
-# Welche Kenntnisse, Fähigkeiten und Fertigkeiten sind für eine
-# erfolgreiche Teilnahme vorauszusetzen?
-# 
-# Alternativ können die Module benannt werden welche für die erfolgreiche 
-# Teilnahme im Vorfeld zu belegen sind.
-
-prerequisites: 
-    type: str
-    label: {
-        de: "Voraussetzungen für die Teilnahme",
-        en: "prerequisites"
-    }
-
-#
-# Literatur und multimediale Lehr- und Lernprogramme
-#
-# 
-# Wie können die Studierenden sich auf die Teilnahme an diesem Modul vorbereiten?
-# 
-teaching-material:     
-    type: str
-    label: {
-        de: "Literatur und multimediale Lehr- und Lernprogramme",
-        en: "media of instruction"
-    }
-
-#
-# Lehrbriefautor
-#
-author-of-indenture: 
-    type: str
-    label: {
-        de: "Lehrbriefautor",
-        en: "author of indenture"
-    }
-
-#
-# Verwendung in (Studienprogramm)
-#
-used-in:
-    type: str
-    label: {
-        de: "Verwendung",
-        en: "used in study programs"
-    }
-
-#
-# Arbeitsaufwand
-#
-workload:
-    type: str
-    label: {
-        de: "Arbeitsaufwand / Gesamtworkload",
-        en: "workload"
-    }
-#
-# credits/ECTS
-#
-credits:
-    type: num 
-    label: { 
-        en: "credits and weight of mark",
-        de: "Kreditpunkte und Gewichtung der Note in der Gesamtnote"
-    }
-    template: 
-        de: "${value}CP Gewichtung: ${value}CP von 120CP "
-        en: "${value}CP weight: ${value} / 120 "    
-    
-
-#
-# Leistungsnachweis
-#
-form-of-exam: 
-    label: {
-        de: "Leistungsnachweis",
-        en: "form of examination"
-    }
-    type: enum 
-    values: {
-        'written' : {
-            de: "Schriftliche Prüfung",
-            en: "written exam"
-        },
-        'oral' : { 
-            de: "Mündliche Prüfung",
-            en: "oral exam"
-        },
-        'alternative' : {
-            de: "Alternative Prüfungunsleistung",
-            en: "alternative examination"
-        }
-    }
-    spec: true
-    template:
-        de: "${value} (${spec})" 
-        en: "${value} (${spec})"
-
-
-#
-# Semester
-#
-term: 
-    label: {
-        de: "Semester",
-        en: "term"
-    }
-    type: multinum
-    template: 
-        de: " ${value}. Semester"
-        en: " ${value}. semester"
-
-#
-# Häufigkeit des Angebots
-#
-frequency: 
-    label: {
-        de: "Häufigkeit des Angebots",
-        en: "frequency of Offer"
-    }
-    type: "enum"
-    values: {
-        'once_per_term' : {
-            de: "jedes Semester",
-            en: "every term"
-        },
-        'once_per_year' : {
-            de: "einmal im Studienjahr",
-            en: "once per study year"
-        }
-    }
-
-duration: 
-    type: int
-    label:
-        de: Dauer
-        en: duration
-    template: 
-        de: "$value Semester"
-        en: "$value term(s)"
-
-kind:
-    type: enum
-    label: {
-        de: 'Art der Veranstaltung (Pflicht, Wahl, etc.)',
-        en: 'kind of module (compulsory, elective)'
-    }
-    values: {
-        'compulsory': {
-            de: "Pflicht",
-            en: "compulsory"
-        },
-        'elective' : {
-            de: "Wahl/Wahlpflicht",
-            en: "elective"
-        }
-    }
-
-remarks:     
-    type: str
-    label: {
-        de: "Besonderes",
-        en: "remarks"
-    }
-
-
-