From 4fca7c7bae708eab6e7d9e865cc2d37457b4bc9b Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Thu, 9 May 2024 22:57:10 +0200 Subject: [PATCH 01/19] initial stab in restructuring and generating curricula --- coursebuilder/__main__.py | 29 +++-- coursebuilder/metagenerator.py | 3 +- coursebuilder/tablegenerator.py | 2 + requirements.txt | 7 ++ test/Makefile | 38 +++++++ test/{simple => }/book.yaml | 1 + test/{simple => }/fields.yaml | 0 test/{simple => }/mod.cg.yaml | 0 test/{simple => }/mod.interactsys.yaml | 0 test/mod.test.yaml | 103 ++++++++++++++++++ .../pandoc-template/eisvogel.latex | 0 test/{simple => }/schema.yaml | 0 test/simple/Makefile | 28 ----- test/simple/mod.test.yaml | 98 ----------------- 14 files changed, 172 insertions(+), 137 deletions(-) create mode 100644 requirements.txt create mode 100644 test/Makefile rename test/{simple => }/book.yaml (96%) rename test/{simple => }/fields.yaml (100%) rename test/{simple => }/mod.cg.yaml (100%) rename test/{simple => }/mod.interactsys.yaml (100%) create mode 100644 test/mod.test.yaml rename test/{simple => }/pandoc-template/eisvogel.latex (100%) rename test/{simple => }/schema.yaml (100%) delete mode 100644 test/simple/Makefile delete mode 100644 test/simple/mod.test.yaml diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 996e426..bc36a3d 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -11,16 +11,17 @@ actual values are kept in YAML files in order to version them with git. """ from argparse import ArgumentParser -import yaml import string import os,sys +import yaml +import pandas as pd + from tablegenerator import TableGenerator from markdowngenerator import MarkdownGenerator from templategenerator import TemplateGenerator from metagenerator import MetaGenerator - class CourseBuilder: @staticmethod @@ -39,7 +40,10 @@ class CourseBuilder: parser.add_argument('--level',type=int,default=1,help="level of header tags") parser.add_argument('--table-gen',type=str,default=None,help='runs table generator') parser.add_argument('--template',type=str,default=None,help='defines a template to be used with fields') - + parser.add_argument('-o','--out',type=str,default=None,help='set the output type') + + parser.add_argument('--maxcol',type=int,default=28,help='maximum size of left column') + # get arguments args = parser.parse_args() @@ -114,14 +118,21 @@ class CourseBuilder: generator = MetaGenerator() generator.set_schema(actual_schema) - table_items = generator.process(yaml.load(fm,Loader=yaml.Loader),fields=actual_fields,lang=args.lang,pagebreak=args.pagebreak,createTitle=args.title,header_level=args.level,template=args.template) + meta = yaml.load(fm,Loader=yaml.Loader) - if args.template: - TemplateGenerator.generate(table_items) - else: - MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) + table_items = generator.process(meta=meta,fields=actual_fields,lang=args.lang) - # print(table_items) + df = pd.DataFrame(table_items) + df.columns = df.iloc[0] + df = df[1:] + print(df.to_markdown(tablefmt='grid', index=False, maxcolwidths=[args.maxcol,None])) + print('\n') + + if args.pagebreak: + print('\\pagebreak') + + + # MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) else: parser.print_help() diff --git a/coursebuilder/metagenerator.py b/coursebuilder/metagenerator.py index fe70f33..4d3ef52 100644 --- a/coursebuilder/metagenerator.py +++ b/coursebuilder/metagenerator.py @@ -1,5 +1,4 @@ import os,string,sys -import yaml class MetaGenerator: @@ -92,7 +91,7 @@ class MetaGenerator: return [k,', '.join(parts)] - def process(self,meta,fields = [],lang = 'de',pagebreak = False,createTitle=False,header_level=1,template=None): + def process(self,meta,fields = [],lang = 'de'): table_items = [] diff --git a/coursebuilder/tablegenerator.py b/coursebuilder/tablegenerator.py index 3bdc6f5..824e714 100644 --- a/coursebuilder/tablegenerator.py +++ b/coursebuilder/tablegenerator.py @@ -4,6 +4,8 @@ import string import tempfile import subprocess import os +import pandas as pd +import tabulate class TableGenerator: """ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e0f7ee4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +numpy==1.26.4 +pandas==2.2.2 +python-dateutil==2.9.0.post0 +pytz==2024.1 +six==1.16.0 +tabulate==0.9.0 +tzdata==2024.1 diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..1a11f91 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,38 @@ + +build_dir := build +target_en := ${build_dir}/table.en.pdf +target_de := ${build_dir}/table.de.pdf + +target_flags := --template pandoc-template/eisvogel.latex + +coursebuilder := ../coursebuilder + +${target_en}: + @echo "creating English version ..." + mkdir -p ${build_dir} + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l en -f fields.yaml | pandoc ${target_flags} -o ${target_en} + +${target_de}: + @echo "creating German version ..." + mkdir -p ${build_dir} + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de} + +all: ${target_de} ${target_en} + +clean: + rm ${target_de} ${target_en} + +# debug-template: +# python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits goal content --template "$$name | $$credits" + +# debug-markdown: +# python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits + +# debug-book: +# python ${coursebuilder} -s schema.yaml -b book.yaml -l de + +debug: + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -l de -f name credits goal content | pandoc ${target_flags} -V lang:de -o ${target_de} + + +.PHONY: clean \ No newline at end of file diff --git a/test/simple/book.yaml b/test/book.yaml similarity index 96% rename from test/simple/book.yaml rename to test/book.yaml index 5ec66d5..38011e9 100644 --- a/test/simple/book.yaml +++ b/test/book.yaml @@ -33,5 +33,6 @@ book: en: "## elective courses {.unnumbered}" - modules: - mod.interactsys.yaml + - mod.test.yaml diff --git a/test/simple/fields.yaml b/test/fields.yaml similarity index 100% rename from test/simple/fields.yaml rename to test/fields.yaml diff --git a/test/simple/mod.cg.yaml b/test/mod.cg.yaml similarity index 100% rename from test/simple/mod.cg.yaml rename to test/mod.cg.yaml diff --git a/test/simple/mod.interactsys.yaml b/test/mod.interactsys.yaml similarity index 100% rename from test/simple/mod.interactsys.yaml rename to test/mod.interactsys.yaml diff --git a/test/mod.test.yaml b/test/mod.test.yaml new file mode 100644 index 0000000..c70d8ca --- /dev/null +++ b/test/mod.test.yaml @@ -0,0 +1,103 @@ +name: + de: Test Vorlesung + en: Lecture of Test + + +id: + value: Test + +credits: + value: 5 + +form-of-exam: + value: written + +form-of-instruction: + value: { 'lecture': 2, 'exersise': 1 } + +term: + value: [1, 3] + +duration: + value: 1 + +kind: + value: compulsory + +goal: + de: | + **What is it** + + Lorem Ipsum is simply dummy text of the printing and typesetting + industry. Lorem Ipsum has been the industry's standard dummy text + ever since the 1500s, when an unknown printer took a galley of type + and scrambled it to make a type specimen book. It has survived not only + five centuries, but also the leap into electronic typesetting, remaining + essentially unchanged. It was popularised in the 1960s with the release + of Letraset sheets containing Lorem Ipsum passages, and more recently with + desktop publishing software like Aldus PageMaker including versions of + Lorem Ipsum. + + + + en: | + + +content: + de: | + **Where did it come from** + + Contrary to popular belief, Lorem Ipsum is not simply random text. + It has roots in a piece of classical Latin literature from 45 BC, + making it over 2000 years old. Richard McClintock, a Latin professor + at Hampden-Sydney College in Virginia, looked up one of the more + obscure Latin words, consectetur, from a Lorem Ipsum passage, and + going through the cites of the word in classical literature, + discovered the undoubtable source. Lorem Ipsum comes from sections + 1.10.32 and 1.10.33 of "de Finibus Bonorum et Malorum" (The + Extremes of Good and Evil) by Cicero, written in 45 BC. This book + is a treatise on the theory of ethics, very popular during the + Renaissance. The first line of Lorem Ipsum, "Lorem ipsum dolor + sit amet..", comes from a line in section 1.10.32. + + + en: | + + +teaching-material: + de: | + + en: | + + +prerequisites: + de: "" + en: "" + +author-of-indenture: + de: "" + en: "" + +used-in: + de: "Master Applied Computerscience" + en: "Master Applied Computerscience" + +workload: + de: "2SWS Vorlesung 1SWS Übung" + en: "2SWS lecture 1SWS exersise" + +form-of-exam: + value: written + spec: + de: "120min Klausur" + en: "120min exam" + +frequency: + value: once_per_year + +kind: + value: compulsory + +remarks: + de: + en: diff --git a/test/simple/pandoc-template/eisvogel.latex b/test/pandoc-template/eisvogel.latex similarity index 100% rename from test/simple/pandoc-template/eisvogel.latex rename to test/pandoc-template/eisvogel.latex diff --git a/test/simple/schema.yaml b/test/schema.yaml similarity index 100% rename from test/simple/schema.yaml rename to test/schema.yaml diff --git a/test/simple/Makefile b/test/simple/Makefile deleted file mode 100644 index 3047ab0..0000000 --- a/test/simple/Makefile +++ /dev/null @@ -1,28 +0,0 @@ - -coursebuilder := ../../coursebuilder - -table.en.pdf: - @echo "creating English version ..." - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l en -f fields.yaml | pandoc --template pandoc-template/eisvogel.latex -o table.en.pdf - -table.de.pdf: - @echo "creating German version ..." - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc --template pandoc-template/eisvogel.latex -o table.de.pdf - -all: table.en.pdf table.de.pdf - -clean: - rm -f table.en.pdf table.de.pdf - -debug-template: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits --template "$$name | $$credits" - -debug-markdown: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits - -debug-book: - python ${coursebuilder} -s schema.yaml -b book.yaml -l de - -debug: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml - diff --git a/test/simple/mod.test.yaml b/test/simple/mod.test.yaml deleted file mode 100644 index d600dda..0000000 --- a/test/simple/mod.test.yaml +++ /dev/null @@ -1,98 +0,0 @@ - -name: - en: Test Course - - -test: - - competency-table: - de: - - "Lineare Algebra": 'ABC' - - "Vector Spaces": 'A' - - - -# -# nested lists seem to work in Markdown only in the US style way -# -# reference here: https://meta.stackexchange.com/questions/85474/how-to-write-nested-numbered-lists -# -# note the parser actually corrects 'Tervuren' to 3 in resulting data -# - -content: - en: | - 1. Blah - - 2. Blub - - 1. Blah - - 1. Blub - - 1. Blah - - 1. Blub - - 1. Blah - - 1. Blub - - 1. Blah - - 1. Blub - - 3. Blah - - 4. Blub - - - - 5. Blah - - 6. Blah and Blub - - 1. Blah - - 1. Blub - - 7. Blah and Blub - - - Blah - - - Blub - - - Blah - - - Blub - - 8. Blub and Blah - - - Blah - - - Blub - - - Blah - - - Blub - - - Blah - - - Blub - - - Blah - - - Blub - - 9. Blah, Blub and Blub - - - Blah - - - Blub - - - Blah - - - Blub - - - From 85abfeb7435aabf8d8bffdd0e67decbd1ad6997f Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Thu, 9 May 2024 23:38:20 +0200 Subject: [PATCH 02/19] repurpose title to make it configurable from outside --- coursebuilder/__main__.py | 20 ++++++-- coursebuilder/{metagenerator.py => parser.py} | 46 +------------------ test/Makefile | 11 +---- test/schema.yaml | 6 +-- 4 files changed, 21 insertions(+), 62 deletions(-) rename coursebuilder/{metagenerator.py => parser.py} (69%) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index bc36a3d..1c55f5d 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -20,7 +20,7 @@ import pandas as pd from tablegenerator import TableGenerator from markdowngenerator import MarkdownGenerator from templategenerator import TemplateGenerator -from metagenerator import MetaGenerator +from parser import Parser class CourseBuilder: @@ -35,7 +35,7 @@ class CourseBuilder: parser.add_argument('-f','--fields',help="Fields to be used, the table will be build accordingly",action="extend", nargs="+", type=str) parser.add_argument('-s','--schema',help="using provided schema") parser.add_argument('-p','--pagebreak',action="store_true",help="add a pagebreak after each module") - parser.add_argument('-t','--title',action="store_true",help="take first value in list as title") + parser.add_argument('--title',type=str,default=None,help="template for title") parser.add_argument('-b','--book',type=str,help="process a whole curriculum book with sections") parser.add_argument('--level',type=int,default=1,help="level of header tags") parser.add_argument('--table-gen',type=str,default=None,help='runs table generator') @@ -59,7 +59,7 @@ class CourseBuilder: # book mode with predefined setting from a book file if args.book and args.schema: - generator = MetaGenerator() + generator = Parser() with open(args.schema) as sf: generator.set_schema(yaml.load(sf,Loader=yaml.Loader)) @@ -115,18 +115,28 @@ class CourseBuilder: for m in args.meta: with open(m) as fm: - generator = MetaGenerator() + generator = Parser() generator.set_schema(actual_schema) meta = yaml.load(fm,Loader=yaml.Loader) table_items = generator.process(meta=meta,fields=actual_fields,lang=args.lang) + # TODO - something more processable for Pandas? + # return [ { name: Computergraphik }, keys = { name: Modulname } ] + + # get the dataframe df = pd.DataFrame(table_items) + + # use first column for df.columns = df.iloc[0] df = df[1:] + + if args.title != None: + print(args.title.format(df.columns[1]),'\n') + print(df.to_markdown(tablefmt='grid', index=False, maxcolwidths=[args.maxcol,None])) - print('\n') + print('\n') # always add a newline after the table if args.pagebreak: print('\\pagebreak') diff --git a/coursebuilder/metagenerator.py b/coursebuilder/parser.py similarity index 69% rename from coursebuilder/metagenerator.py rename to coursebuilder/parser.py index 4d3ef52..2b59e11 100644 --- a/coursebuilder/metagenerator.py +++ b/coursebuilder/parser.py @@ -1,6 +1,6 @@ import os,string,sys -class MetaGenerator: +class Parser: def __init__(self) -> None: self.__schema = None @@ -109,46 +109,4 @@ class MetaGenerator: print(field,' not resolvable in ',self.__schema,exp) # maybe return tableitems as np.Dataframe? - return table_items - - - - # if template != None: - # # use template generator - # TemplateGenerator.generate(table_items,pagebreak,createTitle,header_level=header_level) - # pass - # else: - # # conventional MD mode - # MarkdownGenerator.generate(table_items,pagebreak,createTitle,header_level=header_level) - - # def process_book_section(self,section,lang='de'): - # pass - - # book mode - # def process_book(self,book,bookpath,create_title,pagebreak,lang='de',header_level=2): - - # actual_fields = [] - - # for bi in book['book']: - # if 'fields' in bi: - # actual_fields = bi['fields'] - # if 'sections' in bi: - # for section in bi['sections']: - # if 'text' in section: - # print(section['text'][lang]) - # if 'modules' in section: - # for m in section['modules']: - # mod_path = os.path.join(os.path.dirname(bookpath),m) - - # with open(mod_path) as fm: - # try: - # table_items = self.process(yaml.load(fm,Loader=yaml.Loader),fields=actual_fields,lang=lang,pagebreak=pagebreak,createTitle=create_title,header_level=header_level) - - # print(table_items) - - # except Exception as exc: - # print(f'{type(exc).__name__} in {mod_path}: {exc}',file=sys.stderr) - - - - + return table_items \ No newline at end of file diff --git a/test/Makefile b/test/Makefile index 1a11f91..c83469e 100644 --- a/test/Makefile +++ b/test/Makefile @@ -22,17 +22,8 @@ all: ${target_de} ${target_en} clean: rm ${target_de} ${target_en} -# debug-template: -# python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits goal content --template "$$name | $$credits" - -# debug-markdown: -# python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -l de -f name credits - -# debug-book: -# python ${coursebuilder} -s schema.yaml -b book.yaml -l de - debug: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -l de -f name credits goal content | pandoc ${target_flags} -V lang:de -o ${target_de} + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --title "## {}" -l de -f name credits goal content | pandoc ${target_flags} -V lang:de -o ${target_de} .PHONY: clean \ No newline at end of file diff --git a/test/schema.yaml b/test/schema.yaml index 8d54fab..e38783c 100644 --- a/test/schema.yaml +++ b/test/schema.yaml @@ -83,11 +83,11 @@ content: # Es sind nur Werte aus der Prüfungsordung zugelassen # form-of-instruction: + type: multikey label: { de: "Lehrform(en)", en: "form of instruction" } - type: multikey keys: { 'lecture' : { de: "Vorlesung", @@ -188,8 +188,8 @@ credits: de: "Kreditpunkte und Gewichtung der Note in der Gesamtnote" } template: - de: "${value}CP Gewichtung: ${value}CP von 120CP " - en: "${value}CP weight: ${value} / 120 " + de: "${value}CP, Gewichtung: ${value}CP von 120CP " + en: "${value}CP, weight: ${value} / 120 " # From e9407a6b6e55907750dee4d7d5c3c8d7cbdc1d66 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Sun, 12 May 2024 21:07:57 +0200 Subject: [PATCH 03/19] avoid "parser" name ... --- coursebuilder/__main__.py | 21 ++++++++++----------- coursebuilder/{parser.py => converter.py} | 5 ++--- test/Makefile | 10 +++++++--- 3 files changed, 19 insertions(+), 17 deletions(-) rename coursebuilder/{parser.py => converter.py} (98%) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 1c55f5d..5f23d02 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -16,11 +16,10 @@ import os,sys import yaml import pandas as pd - from tablegenerator import TableGenerator from markdowngenerator import MarkdownGenerator from templategenerator import TemplateGenerator -from parser import Parser +from converter import Converter class CourseBuilder: @@ -35,7 +34,7 @@ class CourseBuilder: parser.add_argument('-f','--fields',help="Fields to be used, the table will be build accordingly",action="extend", nargs="+", type=str) parser.add_argument('-s','--schema',help="using provided schema") parser.add_argument('-p','--pagebreak',action="store_true",help="add a pagebreak after each module") - parser.add_argument('--title',type=str,default=None,help="template for title") + parser.add_argument('--title',type=str,default=None,help="template for title - use curly brackets (i.e. {}) to mark where the title string is inserted") parser.add_argument('-b','--book',type=str,help="process a whole curriculum book with sections") parser.add_argument('--level',type=int,default=1,help="level of header tags") parser.add_argument('--table-gen',type=str,default=None,help='runs table generator') @@ -49,17 +48,15 @@ class CourseBuilder: if args.table_gen: - tg = TableGenerator() - + tg = TableGenerator() tg.generate_table(args.table_gen) return - # book mode with predefined setting from a book file if args.book and args.schema: - generator = Parser() + generator = Converter() with open(args.schema) as sf: generator.set_schema(yaml.load(sf,Loader=yaml.Loader)) @@ -99,6 +96,7 @@ class CourseBuilder: # get actual fields actual_fields = [] + # use a file instead of list if os.path.isfile(args.fields[0]): with open(args.fields[0]) as ff: actual_fields = yaml.load(ff,Loader=yaml.Loader)['fields'] @@ -115,7 +113,7 @@ class CourseBuilder: for m in args.meta: with open(m) as fm: - generator = Parser() + generator = Converter() generator.set_schema(actual_schema) meta = yaml.load(fm,Loader=yaml.Loader) @@ -123,7 +121,10 @@ class CourseBuilder: table_items = generator.process(meta=meta,fields=actual_fields,lang=args.lang) # TODO - something more processable for Pandas? - # return [ { name: Computergraphik }, keys = { name: Modulname } ] + draft = [ + { 'name' : 'Computergrafik' }, + { 'credits' : '5 ECTS' }, + ] # get the dataframe df = pd.DataFrame(table_items) @@ -140,8 +141,6 @@ class CourseBuilder: if args.pagebreak: print('\\pagebreak') - - # MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) else: diff --git a/coursebuilder/parser.py b/coursebuilder/converter.py similarity index 98% rename from coursebuilder/parser.py rename to coursebuilder/converter.py index 2b59e11..668dab8 100644 --- a/coursebuilder/parser.py +++ b/coursebuilder/converter.py @@ -1,6 +1,6 @@ -import os,string,sys +import string -class Parser: +class Converter: def __init__(self) -> None: self.__schema = None @@ -32,7 +32,6 @@ class Parser: def process_str(self,meta,field,lang='de'): if self.is_translatable(field): - return [self.process_label(field,lang),meta[field][lang]] else: if not 'value' in meta[field]: diff --git a/test/Makefile b/test/Makefile index c83469e..cc6f0ff 100644 --- a/test/Makefile +++ b/test/Makefile @@ -3,10 +3,14 @@ build_dir := build target_en := ${build_dir}/table.en.pdf target_de := ${build_dir}/table.de.pdf +targets := ${target_de} ${target_en} + target_flags := --template pandoc-template/eisvogel.latex coursebuilder := ../coursebuilder +all: ${targets} + ${target_en}: @echo "creating English version ..." mkdir -p ${build_dir} @@ -17,13 +21,13 @@ ${target_de}: mkdir -p ${build_dir} python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de} -all: ${target_de} ${target_en} clean: - rm ${target_de} ${target_en} + rm -f ${targets} debug: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --title "## {}" -l de -f name credits goal content | pandoc ${target_flags} -V lang:de -o ${target_de} + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --title "## {}" -l de -f name credits goal content + # | pandoc ${target_flags} -V lang:de -o ${target_de} .PHONY: clean \ No newline at end of file From df1cff80d80ffabf1cd87ea49044a4b96a85d774 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Thu, 16 May 2024 08:40:09 +0200 Subject: [PATCH 04/19] refactoring previous Markdown generator into a legacy mode --- coursebuilder/__main__.py | 23 ++++++--------- coursebuilder/markdowngenerator.py | 46 ++++++++++++++++++++++-------- test/Makefile | 4 +++ 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 5f23d02..a26a6dc 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -11,7 +11,6 @@ actual values are kept in YAML files in order to version them with git. """ from argparse import ArgumentParser -import string import os,sys import yaml import pandas as pd @@ -40,6 +39,8 @@ class CourseBuilder: parser.add_argument('--table-gen',type=str,default=None,help='runs table generator') parser.add_argument('--template',type=str,default=None,help='defines a template to be used with fields') parser.add_argument('-o','--out',type=str,default=None,help='set the output type') + parser.add_argument('--legacy',action="store_true",help="use legacy generator mode for compatibility") + parser.add_argument('--maxcol',type=int,default=28,help='maximum size of left column') @@ -122,25 +123,19 @@ class CourseBuilder: # TODO - something more processable for Pandas? draft = [ - { 'name' : 'Computergrafik' }, + { 'name' : 'Modulname', 'lang' : 'de' }, { 'credits' : '5 ECTS' }, ] - # get the dataframe - df = pd.DataFrame(table_items) + draft_2 = { + } - # use first column for - df.columns = df.iloc[0] - df = df[1:] + if args.legacy: + MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) + else: + MarkdownGenerator.generate_table(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) - if args.title != None: - print(args.title.format(df.columns[1]),'\n') - print(df.to_markdown(tablefmt='grid', index=False, maxcolwidths=[args.maxcol,None])) - print('\n') # always add a newline after the table - - if args.pagebreak: - print('\\pagebreak') # MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) else: diff --git a/coursebuilder/markdowngenerator.py b/coursebuilder/markdowngenerator.py index 59f4323..c765dab 100644 --- a/coursebuilder/markdowngenerator.py +++ b/coursebuilder/markdowngenerator.py @@ -1,25 +1,47 @@ #!/usr/bin/env python -import textwrap,itertools +import itertools -# we need raw value maybe add a third item - tuple the input? -# alternative use a dictionary -# { name: XYZ } - -# or make it the class class MarkdownGenerator: @staticmethod def generate_tablerow() -> str: pass + + + @staticmethod + def generate_table(table_items,add_pagebreak = False,title_template = None,first_colwidth = 28): + + import pandas as pd + import tabulate + + # get the dataframe + df = pd.DataFrame(table_items) + + # use first column for + df.columns = df.iloc[0] + df = df[1:] + + if title_template != None: + print(title_template.format(df.columns[1]),'\n') + + print(df.to_markdown(tablefmt='grid', index=False, maxcolwidths=[first_colwidth,None])) + print('\n') # always add a newline after the table + + if add_pagebreak: + print('\\pagebreak') + + @staticmethod - def generate(ti,pagebreak = False,title = False,header_level = 1) -> str: + def generate_table_legacy(table_items,add_pagebreak = False,title_template = None,first_colwidth = 28): + + import textwrap line_length = 128 - column_ratio= 0.28 + column_ratio = float(first_colwidth) / 100 h_len = int(line_length * column_ratio) d_len = line_length-h_len @@ -27,8 +49,8 @@ class MarkdownGenerator: # # generate title (currently the first one) # - if title: - print('#' * header_level,ti[0][1],'\n') + if title_template != None: + print(title_template.format(table_items[0][1]),'\n') print(''.join(['+',"".ljust(h_len,'-'),'+',"".ljust(d_len,'-'),'+'])) @@ -40,7 +62,7 @@ class MarkdownGenerator: # test if this affected by a third item! - for k,v in ti: + for k,v in table_items: # if v == None: @@ -79,5 +101,5 @@ class MarkdownGenerator: # to control pagebreaks for pandoc - if pagebreak: + if add_pagebreak: print('\n\\newpage') \ No newline at end of file diff --git a/test/Makefile b/test/Makefile index cc6f0ff..b605d0b 100644 --- a/test/Makefile +++ b/test/Makefile @@ -29,5 +29,9 @@ debug: python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --title "## {}" -l de -f name credits goal content # | pandoc ${target_flags} -V lang:de -o ${target_de} +debug-legacy: + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --legacy --title "## {}" -l de -f name credits goal content + # | pandoc ${target_flags} -V lang:de -o ${target_de} + .PHONY: clean \ No newline at end of file From 52c3ab5c37548a23e9098c839ce92198c1b84698 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Thu, 16 May 2024 17:28:23 +0200 Subject: [PATCH 05/19] final refactor for book mode --- coursebuilder/__main__.py | 105 +++++++++++++++++--------------------- test/Makefile | 9 ++-- 2 files changed, 52 insertions(+), 62 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index a26a6dc..eb3f5bf 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -22,6 +22,45 @@ from converter import Converter class CourseBuilder: + @staticmethod + def generate(args): + if args.schema and args.meta and len(args.fields) > 0: + + # get actual fields + actual_fields = [] + + # use a file instead of list + if os.path.isfile(args.fields[0]): + with open(args.fields[0]) as ff: + actual_fields = yaml.load(ff,Loader=yaml.Loader)['fields'] + else: + actual_fields = args.fields + + + # get schema + actual_schema = None + with open(args.schema) as f: + actual_schema = yaml.load(f,Loader=yaml.Loader) + + # iterate through meta files + for m in args.meta: + with open(m) as fm: + + generator = Converter() + generator.set_schema(actual_schema) + + meta = yaml.load(fm,Loader=yaml.Loader) + + table_items = generator.process(meta=meta,fields=actual_fields,lang=args.lang) + + if args.legacy: + MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) + else: + MarkdownGenerator.generate_table(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) + + + # MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) + @staticmethod def run(): @@ -77,67 +116,17 @@ class CourseBuilder: if 'text' in section: print(section['text'][args.lang]) if 'modules' in section: - for m in section['modules']: - mod_path = os.path.join(os.path.dirname(book_path),m) - - with open(mod_path) as fm: - try: - - table_items = generator.process(yaml.load(fm,Loader=yaml.Loader),fields=actual_fields,lang=args.lang,pagebreak=args.pagebreak,createTitle=args.title,header_level=args.level) - - MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) - - except Exception as exc: - print(f'{type(exc).__name__} in {mod_path}: {exc}',file=sys.stderr) + args.fields = actual_fields + + # expand filenames to be relative to the book + args.meta = [os.path.join(os.path.dirname(book_path),mod_path) for mod_path in section['modules']] + + CourseBuilder.generate(args=args) # verbose command line mode - elif args.schema and args.meta and len(args.fields) > 0: - - # get actual fields - actual_fields = [] - - # use a file instead of list - if os.path.isfile(args.fields[0]): - with open(args.fields[0]) as ff: - actual_fields = yaml.load(ff,Loader=yaml.Loader)['fields'] - else: - actual_fields = args.fields - - - # get schema - actual_schema = None - with open(args.schema) as f: - actual_schema = yaml.load(f,Loader=yaml.Loader) - - # iterate through meta files - for m in args.meta: - with open(m) as fm: - - generator = Converter() - generator.set_schema(actual_schema) - - meta = yaml.load(fm,Loader=yaml.Loader) - - table_items = generator.process(meta=meta,fields=actual_fields,lang=args.lang) - - # TODO - something more processable for Pandas? - draft = [ - { 'name' : 'Modulname', 'lang' : 'de' }, - { 'credits' : '5 ECTS' }, - ] - - draft_2 = { - } - - if args.legacy: - MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) - else: - MarkdownGenerator.generate_table(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) - - - # MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) - + elif args.schema: + CourseBuilder.generate(args=args) else: parser.print_help() diff --git a/test/Makefile b/test/Makefile index b605d0b..73a600a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -2,8 +2,9 @@ build_dir := build target_en := ${build_dir}/table.en.pdf target_de := ${build_dir}/table.de.pdf +target_de_book := ${build_dir}/curricullum.de.pdf -targets := ${target_de} ${target_en} +targets := ${target_de} ${target_en} ${target_de_book} target_flags := --template pandoc-template/eisvogel.latex @@ -29,9 +30,9 @@ debug: python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --title "## {}" -l de -f name credits goal content # | pandoc ${target_flags} -V lang:de -o ${target_de} -debug-legacy: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --legacy --title "## {}" -l de -f name credits goal content - # | pandoc ${target_flags} -V lang:de -o ${target_de} + +debug-book: + python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --legacy | pandoc ${target_flags} -V lang:de -o ${target_de_book} .PHONY: clean \ No newline at end of file From bee767eb985dcbd3e60490a2eeffc50df4ac4d68 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Thu, 16 May 2024 23:15:27 +0200 Subject: [PATCH 06/19] first MVP to separate transformation from representation --- coursebuilder/__main__.py | 54 +++++++++++++-------- coursebuilder/markdowngenerator.py | 3 -- coursebuilder/query.py | 13 +++++ coursebuilder/{converter.py => schema.py} | 59 ++++++++++++++++++++--- test/Makefile | 10 ++-- test/book.yaml | 20 +++++++- test/fields.yaml | 3 +- test/mod.cg.yaml | 3 ++ test/mod.test.yaml | 3 ++ test/schema.yaml | 24 ++++++--- 10 files changed, 146 insertions(+), 46 deletions(-) create mode 100644 coursebuilder/query.py rename coursebuilder/{converter.py => schema.py} (69%) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index eb3f5bf..beedcb5 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -18,48 +18,58 @@ import pandas as pd from tablegenerator import TableGenerator from markdowngenerator import MarkdownGenerator from templategenerator import TemplateGenerator -from converter import Converter +from schema import Schema +from query import Query class CourseBuilder: @staticmethod def generate(args): - if args.schema and args.meta and len(args.fields) > 0: + if args.schema and args.meta: # get actual fields - actual_fields = [] + actual_fields = None # use a file instead of list - if os.path.isfile(args.fields[0]): + if args.fields and os.path.isfile(args.fields[0]): with open(args.fields[0]) as ff: actual_fields = yaml.load(ff,Loader=yaml.Loader)['fields'] else: + # seem we have a list or None actual_fields = args.fields - # get schema - actual_schema = None + schema = None with open(args.schema) as f: - actual_schema = yaml.load(f,Loader=yaml.Loader) + schema = Schema(yaml.load(f,Loader=yaml.Loader)) + + # if no fields are given, take all! + if actual_fields == None: + actual_fields = list(schema.keys()) + + # in case we are running query mode + query = Query(args.query) if args.query else None + + result = [] # iterate through meta files for m in args.meta: with open(m) as fm: - - generator = Converter() - generator.set_schema(actual_schema) meta = yaml.load(fm,Loader=yaml.Loader) - table_items = generator.process(meta=meta,fields=actual_fields,lang=args.lang) + table_items = schema.process(meta=meta,fields=actual_fields,lang=args.lang) if query == None else schema.process_raw(meta=meta,fields=actual_fields,lang=args.lang) if args.legacy: - MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) + MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) + elif query: + query.run(table_items) else: - MarkdownGenerator.generate_table(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.maxcol) + MarkdownGenerator.generate_table(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) - # MarkdownGenerator.generate(table_items,pagebreak=args.pagebreak,title=args.title,header_level=args.level) + + @staticmethod def run(): @@ -71,6 +81,8 @@ class CourseBuilder: parser.add_argument('-l','--lang',help="Language to parse from meta file (use de or en)",default='de') parser.add_argument('-f','--fields',help="Fields to be used, the table will be build accordingly",action="extend", nargs="+", type=str) parser.add_argument('-s','--schema',help="using provided schema") + parser.add_argument('-q','--query',help="compound query to select items") + parser.add_argument('-p','--pagebreak',action="store_true",help="add a pagebreak after each module") parser.add_argument('--title',type=str,default=None,help="template for title - use curly brackets (i.e. {}) to mark where the title string is inserted") parser.add_argument('-b','--book',type=str,help="process a whole curriculum book with sections") @@ -81,7 +93,7 @@ class CourseBuilder: parser.add_argument('--legacy',action="store_true",help="use legacy generator mode for compatibility") - parser.add_argument('--maxcol',type=int,default=28,help='maximum size of left column') + parser.add_argument('--leftcol',type=int,default=28,help='maximum size of left column') # get arguments args = parser.parse_args() @@ -96,11 +108,6 @@ class CourseBuilder: # book mode with predefined setting from a book file if args.book and args.schema: - generator = Converter() - - with open(args.schema) as sf: - generator.set_schema(yaml.load(sf,Loader=yaml.Loader)) - with open(args.book) as bf: actual_fields = [] @@ -109,14 +116,19 @@ class CourseBuilder: book_path = os.path.abspath(args.book) for bi in book['book']: + if 'fields' in bi: actual_fields = bi['fields'] + if 'sections' in bi: for section in bi['sections']: + if 'text' in section: print(section['text'][args.lang]) - if 'modules' in section: + if 'modules' in section: + + # override fields args.fields = actual_fields # expand filenames to be relative to the book diff --git a/coursebuilder/markdowngenerator.py b/coursebuilder/markdowngenerator.py index c765dab..6d45baf 100644 --- a/coursebuilder/markdowngenerator.py +++ b/coursebuilder/markdowngenerator.py @@ -2,8 +2,6 @@ import itertools - - class MarkdownGenerator: @staticmethod @@ -32,7 +30,6 @@ class MarkdownGenerator: if add_pagebreak: print('\\pagebreak') - @staticmethod diff --git a/coursebuilder/query.py b/coursebuilder/query.py new file mode 100644 index 0000000..fab883b --- /dev/null +++ b/coursebuilder/query.py @@ -0,0 +1,13 @@ + +class Query: + + def __init__(self,query) -> None: + self.__query = query + + def run(self,table_items): + # print(table_items) + for row in table_items: + print(row) + # print(eval(self.__query,{row:row})) + pass + diff --git a/coursebuilder/converter.py b/coursebuilder/schema.py similarity index 69% rename from coursebuilder/converter.py rename to coursebuilder/schema.py index 668dab8..78e609a 100644 --- a/coursebuilder/converter.py +++ b/coursebuilder/schema.py @@ -1,13 +1,13 @@ import string -class Converter: +class Schema: - def __init__(self) -> None: - self.__schema = None - - def set_schema(self,schema = None): + def __init__(self,schema) -> None: self.__schema = schema + def keys(self): + return self.__schema.keys() + def get_template(self,field,lang='de'): if 'template' in self.__schema[field]: return self.__schema[field]['template'][lang] @@ -62,7 +62,9 @@ class Converter: t = string.Template(self.get_template(field,lang)) return [self.process_label(field,lang),t.substitute({'value' : v})] + def process_multinum(self,meta,field,lang='de'): + """multinums have various values""" v = meta[field]['value'] t = string.Template(self.get_template(field,lang)) if hasattr(v, "__len__"): @@ -104,8 +106,53 @@ class Converter: case 'int' | 'num' : table_items.append(self.process_num(meta,field,lang)) case 'multinum' : table_items.append(self.process_multinum(meta,field,lang)) case 'multikey': table_items.append(self.process_multikey(meta,field,lang)) + case _: raise ValueError except Exception as exp: print(field,' not resolvable in ',self.__schema,exp) # maybe return tableitems as np.Dataframe? - return table_items \ No newline at end of file + return table_items + + def get_str(self,meta,field,lang='de'): + if self.is_translatable(field): + return meta[field][lang] + else: + if not 'value' in meta[field]: + raise AssertionError(field,'incomplete') + return meta[field]['value'] + + def get_enum(self,meta,field,lang): + vv = meta[field]['value'] + return self.__schema[field]['values'][vv][lang] + + # if self.needs_spec(field): + + # t = string.Template(self.get_template(field=field,lang=lang)) + + # spec = meta[field]['spec'][lang] + + # return [self.process_label(field,lang),t.substitute({'value': enum_val,'spec': spec})] + # else: + # return [self.process_label(field,lang),enum_val] + + + + def get_value(self,meta,field,lang): + match self.__schema[field]['type']: + case 'str': return self.get_str(meta,field,lang) + case 'enum': return self.get_enum(meta,field,lang) + + + def process_raw(self,meta,fields,lang): + + items = [{'field' : field, + 'lang' : lang, + 'type' : self.__schema[field]['type'], + 'label' : self.process_label(field,lang), + 'value' : self.get_value(meta,field,lang) + } + for field in fields] + + + # maybe return tableitems as np.Dataframe? + return items \ No newline at end of file diff --git a/test/Makefile b/test/Makefile index 73a600a..c806cc2 100644 --- a/test/Makefile +++ b/test/Makefile @@ -6,7 +6,7 @@ target_de_book := ${build_dir}/curricullum.de.pdf targets := ${target_de} ${target_en} ${target_de_book} -target_flags := --template pandoc-template/eisvogel.latex +target_flags := --template pandoc-template/eisvogel.latex -V table-use-row-colors:true coursebuilder := ../coursebuilder @@ -22,6 +22,8 @@ ${target_de}: mkdir -p ${build_dir} python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de} +${target_de_book}: + python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 36 | pandoc ${target_flags} -V lang:de -o ${target_de_book} clean: rm -f ${targets} @@ -30,9 +32,7 @@ debug: python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml mod.test.yaml -p --title "## {}" -l de -f name credits goal content # | pandoc ${target_flags} -V lang:de -o ${target_de} - -debug-book: - python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --legacy | pandoc ${target_flags} -V lang:de -o ${target_de_book} - +debug-query: + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "kind == compulsory" .PHONY: clean \ No newline at end of file diff --git a/test/book.yaml b/test/book.yaml index 38011e9..5dc7188 100644 --- a/test/book.yaml +++ b/test/book.yaml @@ -4,8 +4,9 @@ # book: - - fields: - - name + - fields: + - name + - instructor - id - goal - content @@ -36,3 +37,18 @@ book: - mod.test.yaml +# +# tables +# +query: + list-credits-and-workload: + - fields: + - name + - credits + - workload + +# just for ideas +regulations: + - globals: + - course_name: Applied Computer Science (M.Sc.) + \ No newline at end of file diff --git a/test/fields.yaml b/test/fields.yaml index bede013..28aef14 100644 --- a/test/fields.yaml +++ b/test/fields.yaml @@ -1,5 +1,6 @@ fields: - - name + - name + - instructor - id - goal - content diff --git a/test/mod.cg.yaml b/test/mod.cg.yaml index e5b4201..1b614ab 100644 --- a/test/mod.cg.yaml +++ b/test/mod.cg.yaml @@ -2,6 +2,9 @@ name: de: Computergrafik en: Computer Graphics +instructor: + de: Prof. Hartmut Seichter, PhD + en: Prof. Hartmut Seichter, PhD id: value: CG diff --git a/test/mod.test.yaml b/test/mod.test.yaml index c70d8ca..bdfb4f3 100644 --- a/test/mod.test.yaml +++ b/test/mod.test.yaml @@ -2,6 +2,9 @@ name: de: Test Vorlesung en: Lecture of Test +instructor: + de: Cicero + en: Cicero id: value: Test diff --git a/test/schema.yaml b/test/schema.yaml index e38783c..fde6931 100644 --- a/test/schema.yaml +++ b/test/schema.yaml @@ -15,9 +15,8 @@ name: # instructor: type: str - translatable: false label: - de: "Modulverantwortlicher/Modulverantwortliche" + de: "Modulverantwortlicher / Modulverantwortliche" en: "module instructor" @@ -196,11 +195,11 @@ credits: # Leistungsnachweis # form-of-exam: + type: enum label: { de: "Leistungsnachweis", en: "form of examination" } - type: enum values: { 'written' : { de: "Schriftliche Prüfung", @@ -225,28 +224,28 @@ form-of-exam: # Semester # term: + type: multinum label: { de: "Semester", en: "term" } - type: multinum template: - de: " ${value}. Semester" - en: " ${value}. semester" + de: " ${value}\\. Semester" + en: " ${value}\\. semester" # # Häufigkeit des Angebots # frequency: + type: enum label: { de: "Häufigkeit des Angebots", en: "frequency of Offer" } - type: "enum" values: { 'once_per_term' : { de: "jedes Semester", - en: "every term" + en: "every semester" }, 'once_per_year' : { de: "einmal im Studienjahr", @@ -254,6 +253,9 @@ frequency: } } +# +# Dauer des Angebots +# duration: type: int label: @@ -263,6 +265,9 @@ duration: de: "$value Semester" en: "$value term(s)" +# +# Art der Veranstaltung +# kind: type: enum label: { @@ -280,6 +285,9 @@ kind: } } +# +# Freiform Bemerkungen +# remarks: type: str label: { From 0efcea4879f066d9f291f5194833b9a13539599a Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Thu, 16 May 2024 23:20:31 +0200 Subject: [PATCH 07/19] just checking multinum as well --- coursebuilder/schema.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index 78e609a..ee4f058 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -124,6 +124,9 @@ class Schema: def get_enum(self,meta,field,lang): vv = meta[field]['value'] return self.__schema[field]['values'][vv][lang] + + def get_num(self,meta,field,lang): + return meta[field]['value'] # if self.needs_spec(field): @@ -141,6 +144,8 @@ class Schema: match self.__schema[field]['type']: case 'str': return self.get_str(meta,field,lang) case 'enum': return self.get_enum(meta,field,lang) + case 'int' | 'num' : return self.get_num(meta,field,lang) + case 'multinum' : return meta[field]['value'] def process_raw(self,meta,fields,lang): @@ -150,6 +155,7 @@ class Schema: 'type' : self.__schema[field]['type'], 'label' : self.process_label(field,lang), 'value' : self.get_value(meta,field,lang) + } for field in fields] From 833f0bdf4cc24df54f9a0c1e6c8b3ffed69e1aab Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Fri, 17 May 2024 21:04:50 +0200 Subject: [PATCH 08/19] enum method working --- coursebuilder/__main__.py | 2 +- coursebuilder/query.py | 6 ++--- coursebuilder/schema.py | 46 +++++++++++++-------------------------- test/Makefile | 2 +- test/schema.yaml | 3 ++- 5 files changed, 22 insertions(+), 37 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index beedcb5..6a80bb3 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -58,7 +58,7 @@ class CourseBuilder: meta = yaml.load(fm,Loader=yaml.Loader) - table_items = schema.process(meta=meta,fields=actual_fields,lang=args.lang) if query == None else schema.process_raw(meta=meta,fields=actual_fields,lang=args.lang) + table_items = schema.process(meta=meta,fields=actual_fields,lang=args.lang) if query == None else schema.to_dataframe(meta=meta,fields=actual_fields,lang=args.lang) if args.legacy: MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) diff --git a/coursebuilder/query.py b/coursebuilder/query.py index fab883b..17591fa 100644 --- a/coursebuilder/query.py +++ b/coursebuilder/query.py @@ -6,8 +6,8 @@ class Query: def run(self,table_items): # print(table_items) - for row in table_items: - print(row) - # print(eval(self.__query,{row:row})) + for item in table_items: + print(item) + # print(eval(self.__query,locals())) pass diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index ee4f058..80de9b8 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -122,43 +122,27 @@ class Schema: return meta[field]['value'] def get_enum(self,meta,field,lang): - vv = meta[field]['value'] - return self.__schema[field]['values'][vv][lang] - - def get_num(self,meta,field,lang): - return meta[field]['value'] - - # if self.needs_spec(field): - - # t = string.Template(self.get_template(field=field,lang=lang)) - - # spec = meta[field]['spec'][lang] - - # return [self.process_label(field,lang),t.substitute({'value': enum_val,'spec': spec})] - # else: - # return [self.process_label(field,lang),enum_val] - - + enum_val = meta[field]['value'] + return self.__schema[field]['values'][ enum_val ][lang] def get_value(self,meta,field,lang): + """treats receiving the value like a variant, + return values are language specific""" match self.__schema[field]['type']: case 'str': return self.get_str(meta,field,lang) case 'enum': return self.get_enum(meta,field,lang) - case 'int' | 'num' : return self.get_num(meta,field,lang) + case 'int' | 'num' : return meta[field]['value'] case 'multinum' : return meta[field]['value'] - - def process_raw(self,meta,fields,lang): - - items = [{'field' : field, - 'lang' : lang, - 'type' : self.__schema[field]['type'], - 'label' : self.process_label(field,lang), - 'value' : self.get_value(meta,field,lang) - + def to_dataframe(self,meta,fields,lang): + # list comprehension for rows + return [{'field' : field, # field name + 'lang' : lang, # language shortcode + 'type' : self.__schema[field]['type'], # datatype + 'label' : self.process_label(field,lang), # label + 'value' : self.get_value(meta,field,lang), # actual value + 'template' : self.__schema[field]['template'][lang] if 'template' in self.__schema[field] else None, + # getting crazy with nested dict comprehension + 'enum_values' : { k:v[lang] for (k,v) in self.__schema[field]['values'].items()} if 'enum' in self.__schema[field]['type'] else None } for field in fields] - - - # maybe return tableitems as np.Dataframe? - return items \ No newline at end of file diff --git a/test/Makefile b/test/Makefile index c806cc2..f662980 100644 --- a/test/Makefile +++ b/test/Makefile @@ -33,6 +33,6 @@ debug: # | pandoc ${target_flags} -V lang:de -o ${target_de} debug-query: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "kind == compulsory" + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "item['field'] == 'kind' and item['value'] == 'elective'" .PHONY: clean \ No newline at end of file diff --git a/test/schema.yaml b/test/schema.yaml index fde6931..4ceebf5 100644 --- a/test/schema.yaml +++ b/test/schema.yaml @@ -181,7 +181,8 @@ workload: # credits/ECTS # credits: - type: num + type: num + unit: ECTS label: { en: "credits and weight of mark", de: "Kreditpunkte und Gewichtung der Note in der Gesamtnote" From 1381c37500af3e429f0f1e658f88b227e77cd616 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Sat, 18 May 2024 22:26:50 +0200 Subject: [PATCH 09/19] MVP of reworked tuple generation --- coursebuilder/__main__.py | 14 +++++++++++- coursebuilder/query.py | 7 ++++-- coursebuilder/schema.py | 46 +++++++++++++++++++++++---------------- test/schema.yaml | 16 +++++++------- 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 6a80bb3..65b2393 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -58,12 +58,24 @@ class CourseBuilder: meta = yaml.load(fm,Loader=yaml.Loader) - table_items = schema.process(meta=meta,fields=actual_fields,lang=args.lang) if query == None else schema.to_dataframe(meta=meta,fields=actual_fields,lang=args.lang) + table_items = schema.process(meta=meta,fields=actual_fields,lang=args.lang) if query == None else schema.to_list_of_dict(meta=meta,fields=actual_fields,lang=args.lang) if args.legacy: MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) elif query: + query.run(table_items) + + # for i in table_items: + # print(i) + + q = schema.to_list_of_tuple(meta=meta,fields=actual_fields,lang=args.lang) + + for i in q: + print(i) + + # MarkdownGenerator.generate_table(table_items=q,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) + else: MarkdownGenerator.generate_table(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) diff --git a/coursebuilder/query.py b/coursebuilder/query.py index 17591fa..835fe36 100644 --- a/coursebuilder/query.py +++ b/coursebuilder/query.py @@ -1,4 +1,6 @@ +import pandas as pd + class Query: def __init__(self,query) -> None: @@ -6,8 +8,9 @@ class Query: def run(self,table_items): # print(table_items) - for item in table_items: - print(item) + # for item in table_items: + # pass + # print(item) # print(eval(self.__query,locals())) pass diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index 80de9b8..ee730c0 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -92,7 +92,7 @@ class Schema: return [k,', '.join(parts)] - def process(self,meta,fields = [],lang = 'de'): + def process(self,meta,fields,lang): table_items = [] @@ -113,28 +113,15 @@ class Schema: # maybe return tableitems as np.Dataframe? return table_items - def get_str(self,meta,field,lang='de'): - if self.is_translatable(field): - return meta[field][lang] - else: - if not 'value' in meta[field]: - raise AssertionError(field,'incomplete') - return meta[field]['value'] - - def get_enum(self,meta,field,lang): - enum_val = meta[field]['value'] - return self.__schema[field]['values'][ enum_val ][lang] - def get_value(self,meta,field,lang): """treats receiving the value like a variant, return values are language specific""" match self.__schema[field]['type']: - case 'str': return self.get_str(meta,field,lang) - case 'enum': return self.get_enum(meta,field,lang) - case 'int' | 'num' : return meta[field]['value'] - case 'multinum' : return meta[field]['value'] + case 'str': return meta[field][lang] if self.is_translatable(field) else meta[field]['value'] + case 'enum' | 'int' | 'num' | 'multikey' : return meta[field]['value'] + case 'multinum': return meta[field]['value'] if hasattr(meta[field]['value'],'__iter__') else (meta[field]['value'],) # force list! - def to_dataframe(self,meta,fields,lang): + def to_list_of_dict(self,meta,fields,lang): # list comprehension for rows return [{'field' : field, # field name 'lang' : lang, # language shortcode @@ -143,6 +130,27 @@ class Schema: 'value' : self.get_value(meta,field,lang), # actual value 'template' : self.__schema[field]['template'][lang] if 'template' in self.__schema[field] else None, # getting crazy with nested dict comprehension - 'enum_values' : { k:v[lang] for (k,v) in self.__schema[field]['values'].items()} if 'enum' in self.__schema[field]['type'] else None + 'enum_values' : { k:v[lang] for (k,v) in self.__schema[field]['values'].items()} if 'enum' in self.__schema[field]['type'] else None, + 'key_values' : { k:v[lang] for (k,v) in self.__schema[field]['keys'].items()} if 'multikey' in self.__schema[field]['type'] else None, + 'spec' : meta[field]['spec'][lang] if 'spec' in meta[field] else None } for field in fields] + + def to_list_of_tuple(self,meta,fields,lang): + # generate a list of tuples with key and value (text) + list = [] + for r in self.to_list_of_dict(meta,fields,lang): + match r['type']: + case 'str' : + list.append( (r['label'],r['value']) ) + case 'int' | 'num' : + list.append( ( r['label'], r['template'].format(value=r['value'],spec=r['spec']) if r['template'] else r['value']) ) + case 'enum' : + list.append( ( r['label'], r['template'].format(value=r['enum_values'][r['value']],spec=r['spec']) + if r['template'] else r['enum_values'][r['value']] ) ) + case 'multikey' : + list.append( ( r['label'], ', '.join( [r['template'].format(key=r['key_values'][k],value=v) for k,v in r['value'].items()] ) ) ) + case 'multinum' : + list.append( (r['label'], ', '.join( r['template'].format(value=v) for v in r['value'])) ) + + return list \ No newline at end of file diff --git a/test/schema.yaml b/test/schema.yaml index 4ceebf5..94c7820 100644 --- a/test/schema.yaml +++ b/test/schema.yaml @@ -114,8 +114,8 @@ form-of-instruction: } } template: - de: "${key} (${value}SWS)" - en: "${key} (${value}SWS)" + de: "{key} ({value}SWS)" + en: "{key} ({value}SWS)" # # Voraussetzungen für die Teilnahme @@ -188,8 +188,8 @@ credits: de: "Kreditpunkte und Gewichtung der Note in der Gesamtnote" } template: - de: "${value}CP, Gewichtung: ${value}CP von 120CP " - en: "${value}CP, weight: ${value} / 120 " + de: "{value}CP, Gewichtung: {value}CP von 120CP " + en: "{value}CP, weight: {value} / 120 " # @@ -217,8 +217,8 @@ form-of-exam: } spec: true template: - de: "${value} (${spec})" - en: "${value} (${spec})" + de: "{value} ({spec})" + en: "{value} ({spec})" # @@ -263,8 +263,8 @@ duration: de: Dauer en: duration template: - de: "$value Semester" - en: "$value term(s)" + de: "{value} Semester" + en: "{value} term(s)" # # Art der Veranstaltung From 7c73d3b5f6d04dfa55200c68122424ad7f7b6ef9 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Sun, 19 May 2024 10:03:00 +0200 Subject: [PATCH 10/19] disable old processing code --- coursebuilder/__main__.py | 40 +++++------ coursebuilder/schema.py | 136 +++++++++++++++++++------------------- test/Makefile | 2 +- test/schema.yaml | 4 +- 4 files changed, 89 insertions(+), 93 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 65b2393..b55e6e5 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -56,33 +56,28 @@ class CourseBuilder: for m in args.meta: with open(m) as fm: - meta = yaml.load(fm,Loader=yaml.Loader) - - table_items = schema.process(meta=meta,fields=actual_fields,lang=args.lang) if query == None else schema.to_list_of_dict(meta=meta,fields=actual_fields,lang=args.lang) - if args.legacy: - MarkdownGenerator.generate_table_legacy(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) - elif query: - query.run(table_items) - - # for i in table_items: - # print(i) - - q = schema.to_list_of_tuple(meta=meta,fields=actual_fields,lang=args.lang) - - for i in q: - print(i) + MarkdownGenerator.generate_table_legacy( + table_items=schema.to_list_of_tuple( + meta=yaml.load(fm,Loader=yaml.Loader), + fields=actual_fields, + lang=args.lang), + add_pagebreak=args.pagebreak, + title_template=args.title, + first_colwidth=args.leftcol) - # MarkdownGenerator.generate_table(table_items=q,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) - else: - MarkdownGenerator.generate_table(table_items=table_items,add_pagebreak=args.pagebreak,title_template=args.title,first_colwidth=args.leftcol) + MarkdownGenerator.generate_table( + table_items=schema.to_list_of_tuple( + meta=yaml.load(fm,Loader=yaml.Loader), + fields=actual_fields, + lang=args.lang), + add_pagebreak=args.pagebreak, + title_template=args.title, + first_colwidth=args.leftcol) - - - @staticmethod def run(): @@ -104,8 +99,7 @@ class CourseBuilder: parser.add_argument('-o','--out',type=str,default=None,help='set the output type') parser.add_argument('--legacy',action="store_true",help="use legacy generator mode for compatibility") - - parser.add_argument('--leftcol',type=int,default=28,help='maximum size of left column') + parser.add_argument('--leftcol',type=int,default=35,help='maximum size of left column') # get arguments args = parser.parse_args() diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index ee730c0..a81fda7 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -26,92 +26,92 @@ class Schema: else: return False - def process_label(self,field,lang='de'): - # processes the label of a field item - return self.__schema[field]['label'][lang] + # def process_label(self,field,lang='de'): + # # processes the label of a field item + # return self.__schema[field]['label'][lang] - def process_str(self,meta,field,lang='de'): - if self.is_translatable(field): - return [self.process_label(field,lang),meta[field][lang]] - else: - if not 'value' in meta[field]: - raise AssertionError(field,'incomplete') + # def process_str(self,meta,field,lang='de'): + # if self.is_translatable(field): + # return [self.process_label(field,lang),meta[field][lang]] + # else: + # if not 'value' in meta[field]: + # raise AssertionError(field,'incomplete') - return [self.process_label(field,lang),meta[field]['value']] + # return [self.process_label(field,lang),meta[field]['value']] - def process_enum(self,meta,field,lang='de'): - """ - enum have a specification 'specs' option - that can be forced by the scheme - """ - vv = meta[field]['value'] - enum_val = self.__schema[field]['values'][vv][lang] + # def process_enum(self,meta,field,lang='de'): + # """ + # enum have a specification 'specs' option + # that can be forced by the scheme + # """ + # vv = meta[field]['value'] + # enum_val = self.__schema[field]['values'][vv][lang] - if self.needs_spec(field): + # if self.needs_spec(field): - t = string.Template(self.get_template(field=field,lang=lang)) + # t = string.Template(self.get_template(field=field,lang=lang)) - spec = meta[field]['spec'][lang] + # spec = meta[field]['spec'][lang] - return [self.process_label(field,lang),t.substitute({'value': enum_val,'spec': spec})] - else: - return [self.process_label(field,lang),enum_val] + # return [self.process_label(field,lang),t.substitute({'value': enum_val,'spec': spec})] + # else: + # return [self.process_label(field,lang),enum_val] - def process_num(self,meta,field,lang='de'): - v = meta[field]['value'] - t = string.Template(self.get_template(field,lang)) - return [self.process_label(field,lang),t.substitute({'value' : v})] + # def process_num(self,meta,field,lang='de'): + # v = meta[field]['value'] + # t = string.Template(self.get_template(field,lang)) + # return [self.process_label(field,lang),t.substitute({'value' : v})] - def process_multinum(self,meta,field,lang='de'): - """multinums have various values""" - v = meta[field]['value'] - t = string.Template(self.get_template(field,lang)) - if hasattr(v, "__len__"): - vv = [t.substitute({'value' : ev}) for ev in v] - return [self.process_label(field,lang),', '.join(vv)] - else: - return self.process_num(meta=meta,field=field,lang=lang) + # def process_multinum(self,meta,field,lang='de'): + # """multinums have various values""" + # v = meta[field]['value'] + # t = string.Template(self.get_template(field,lang)) + # if hasattr(v, "__len__"): + # vv = [t.substitute({'value' : ev}) for ev in v] + # return [self.process_label(field,lang),', '.join(vv)] + # else: + # return self.process_num(meta=meta,field=field,lang=lang) - def process_multikey(self,meta,field,lang='de'): - """ - multikey need to assign a numeric value to a key - """ - vs = meta[field]['value'] - t = string.Template(self.get_template(field,lang)) + # def process_multikey(self,meta,field,lang='de'): + # """ + # multikey need to assign a numeric value to a key + # """ + # vs = meta[field]['value'] + # t = string.Template(self.get_template(field,lang)) - k = self.process_label(field,lang) + # k = self.process_label(field,lang) - parts = [] + # parts = [] - for e in vs: - kk = self.__schema[field]['keys'][e][lang] - parts.append(t.substitute({'key': kk, 'value' : vs[e]})) + # for e in vs: + # kk = self.__schema[field]['keys'][e][lang] + # parts.append(t.substitute({'key': kk, 'value' : vs[e]})) - return [k,', '.join(parts)] + # return [k,', '.join(parts)] - def process(self,meta,fields,lang): + # def process(self,meta,fields,lang): - table_items = [] + # table_items = [] - # iterate over requested fields - for field in fields: - try: - # correlate with schema and append - match self.__schema[field]['type']: - case 'str': table_items.append(self.process_str(meta,field,lang)) - case 'enum': table_items.append(self.process_enum(meta,field,lang)) - case 'int' | 'num' : table_items.append(self.process_num(meta,field,lang)) - case 'multinum' : table_items.append(self.process_multinum(meta,field,lang)) - case 'multikey': table_items.append(self.process_multikey(meta,field,lang)) - case _: raise ValueError - except Exception as exp: - print(field,' not resolvable in ',self.__schema,exp) + # # iterate over requested fields + # for field in fields: + # try: + # # correlate with schema and append + # match self.__schema[field]['type']: + # case 'str': table_items.append(self.process_str(meta,field,lang)) + # case 'enum': table_items.append(self.process_enum(meta,field,lang)) + # case 'int' | 'num' : table_items.append(self.process_num(meta,field,lang)) + # case 'multinum' : table_items.append(self.process_multinum(meta,field,lang)) + # case 'multikey': table_items.append(self.process_multikey(meta,field,lang)) + # case _: raise ValueError + # except Exception as exp: + # print(field,' not resolvable in ',self.__schema,exp) - # maybe return tableitems as np.Dataframe? - return table_items + # # maybe return tableitems as np.Dataframe? + # return table_items def get_value(self,meta,field,lang): """treats receiving the value like a variant, @@ -126,7 +126,7 @@ class Schema: return [{'field' : field, # field name 'lang' : lang, # language shortcode 'type' : self.__schema[field]['type'], # datatype - 'label' : self.process_label(field,lang), # label + 'label' : self.__schema[field]['label'][lang], # label 'value' : self.get_value(meta,field,lang), # actual value 'template' : self.__schema[field]['template'][lang] if 'template' in self.__schema[field] else None, # getting crazy with nested dict comprehension @@ -137,7 +137,9 @@ class Schema: for field in fields] def to_list_of_tuple(self,meta,fields,lang): - # generate a list of tuples with key and value (text) + """ + generates a list of tuples with a label and value (text) + """ list = [] for r in self.to_list_of_dict(meta,fields,lang): match r['type']: diff --git a/test/Makefile b/test/Makefile index f662980..b449688 100644 --- a/test/Makefile +++ b/test/Makefile @@ -23,7 +23,7 @@ ${target_de}: python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de} ${target_de_book}: - python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 36 | pandoc ${target_flags} -V lang:de -o ${target_de_book} + python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 20 --legacy | pandoc ${target_flags} -V lang:de -o ${target_de_book} clean: rm -f ${targets} diff --git a/test/schema.yaml b/test/schema.yaml index 94c7820..a7cf2f3 100644 --- a/test/schema.yaml +++ b/test/schema.yaml @@ -231,8 +231,8 @@ term: en: "term" } template: - de: " ${value}\\. Semester" - en: " ${value}\\. semester" + de: "{value}\\. Semester" + en: "{value}\\. semester" # # Häufigkeit des Angebots From 18df4d059e874a83f5c9c119a171a176fd60b7f7 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Sun, 19 May 2024 14:03:41 +0200 Subject: [PATCH 11/19] first run of a documentation to describe the use of the various fields --- coursebuilder/docs/quickstart.md | 41 ++++++++++++++++++++++++++++++++ coursebuilder/schema.py | 23 ++++++++++++------ 2 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 coursebuilder/docs/quickstart.md diff --git a/coursebuilder/docs/quickstart.md b/coursebuilder/docs/quickstart.md new file mode 100644 index 0000000..a293ef4 --- /dev/null +++ b/coursebuilder/docs/quickstart.md @@ -0,0 +1,41 @@ +# Concept + +The concept behind coursebuilder is to store curricula descriptions in `YAML` files that can be versioned in a git repository. Unlike classic databases an observable and well defined versioning is paramount in these descriptions as they are the legal foundation for study and exam regulations. + +The following pieces play together here: + +- `schema` files, usually a `schema.yaml` +- `mod` files, usually something along the lines of `mod.coursecode.yaml` +- `book` files describing a whole regulation set and course global details +- some sort of transformation with `coursebuilder` into Markdown that is piped through [pandoc](https://pandoc.org) in order to generate PDF, HTML and other representation from this code + +# schema files + +Schema files are responsible to describe the used structures in a database. The following datatypes are supported: + +- `str` a simple string, can be accompanied with a `template` +- `enum` a classic enum datatype with a fixed set of values +- `num` a numeric datatype +- `multinum` an array type with the possibility to `spec` each value +- `multikey` a key-value type with additional numeric data associated with each key instance + +# mod files (modules) + +Modules describe a course in detail and implement an instance of the schema file. Especially `strings` and `enums` are translatable One of the plan is to use a validator to find inconsistencies automatically, like workloads that are not following the 30h = 1ECTS rule. + + +# datatypes + +## `str` datatype + +```yaml +# this would reside in a schema field on top level +# a field of name 'id' +id: # name of the field + type: str # sets the datatype to str + translatable: false # enforces the value is not translatable (default is true) + label: { # label describes the meaning of the datatype in regards of the schema + de: "Kürzel", # translation of the label in German (de) + en: "code" # translation of the label in English (en) + } +``` \ No newline at end of file diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index a81fda7..0f7612e 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -8,11 +8,11 @@ class Schema: def keys(self): return self.__schema.keys() - def get_template(self,field,lang='de'): - if 'template' in self.__schema[field]: - return self.__schema[field]['template'][lang] - else: - return "$value" + # def get_template(self,field,lang='de'): + # if 'template' in self.__schema[field]: + # return self.__schema[field]['template'][lang] + # else: + # return "$value" def is_translatable(self,field): if 'translatable' in self.__schema[field]: @@ -114,14 +114,20 @@ class Schema: # return table_items def get_value(self,meta,field,lang): - """treats receiving the value like a variant, - return values are language specific""" + """ + treats receiving the value like a variant, + returns values with their language specific representations + """ match self.__schema[field]['type']: case 'str': return meta[field][lang] if self.is_translatable(field) else meta[field]['value'] case 'enum' | 'int' | 'num' | 'multikey' : return meta[field]['value'] case 'multinum': return meta[field]['value'] if hasattr(meta[field]['value'],'__iter__') else (meta[field]['value'],) # force list! def to_list_of_dict(self,meta,fields,lang): + """ + generates a list of dict which can easily be converted + to a pandas dataframe + """ # list comprehension for rows return [{'field' : field, # field name 'lang' : lang, # language shortcode @@ -139,6 +145,9 @@ class Schema: def to_list_of_tuple(self,meta,fields,lang): """ generates a list of tuples with a label and value (text) + this is usually consumed by a Markdown generator + + todo: needs deuglyfication of free standing loop, templates are possible for all """ list = [] for r in self.to_list_of_dict(meta,fields,lang): From e816fe50a2d5dc0cddce6516165927abaf6a4594 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Sun, 19 May 2024 14:08:55 +0200 Subject: [PATCH 12/19] move directory to root --- {coursebuilder/docs => docs}/quickstart.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {coursebuilder/docs => docs}/quickstart.md (100%) diff --git a/coursebuilder/docs/quickstart.md b/docs/quickstart.md similarity index 100% rename from coursebuilder/docs/quickstart.md rename to docs/quickstart.md From e489ef15175ee7ff45f20791d6c4efe782d2f4cf Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Wed, 22 May 2024 19:45:24 +0200 Subject: [PATCH 13/19] minor update --- README.md | 12 +++++++++--- coursebuilder/schema.py | 3 ++- test/Makefile | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 368648b..d017230 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ actual values are kept in YAML files in order to version them with git. ```sh $> python coursebuilder -usage: [-h] [-m META [META ...]] [-l LANG] [-f FIELDS [FIELDS ...]] [-s SCHEMA] [-p] [-t] [-b BOOK] [--level LEVEL] - [--table-gen TABLE_GEN] +usage: [-h] [-m META [META ...]] [-l LANG] [-f FIELDS [FIELDS ...]] [-s SCHEMA] [-q QUERY] [-p] [--title TITLE] [-b BOOK] [--level LEVEL] [--table-gen TABLE_GEN] + [--template TEMPLATE] [-o OUT] [--legacy] [--leftcol LEFTCOL] versatile curricula generator @@ -23,12 +23,18 @@ options: Fields to be used, the table will be build accordingly -s SCHEMA, --schema SCHEMA using provided schema + -q QUERY, --query QUERY + compound query to select items -p, --pagebreak add a pagebreak after each module - -t, --title take first value in list as title + --title TITLE template for title - use curly brackets (i.e. {}) to mark where the title string is inserted -b BOOK, --book BOOK process a whole curriculum book with sections --level LEVEL level of header tags --table-gen TABLE_GEN runs table generator + --template TEMPLATE defines a template to be used with fields + -o OUT, --out OUT set the output type + --legacy use legacy generator mode for compatibility + --leftcol LEFTCOL maximum size of left column ``` # Author diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index 0f7612e..1b0291d 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -164,4 +164,5 @@ class Schema: case 'multinum' : list.append( (r['label'], ', '.join( r['template'].format(value=v) for v in r['value'])) ) - return list \ No newline at end of file + return list + diff --git a/test/Makefile b/test/Makefile index b449688..646a326 100644 --- a/test/Makefile +++ b/test/Makefile @@ -23,7 +23,7 @@ ${target_de}: python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de} ${target_de_book}: - python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 20 --legacy | pandoc ${target_flags} -V lang:de -o ${target_de_book} + python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 25 --legacy | pandoc ${target_flags} -V toc:true -V lang:de -o ${target_de_book} clean: rm -f ${targets} From 7078c8255b79ca92ca23169031014081906722c4 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Mon, 27 May 2024 13:28:22 +0200 Subject: [PATCH 14/19] minor cleanup --- README.md | 7 ++- coursebuilder/__main__.py | 10 ++++- coursebuilder/query.py | 4 ++ coursebuilder/schema.py | 95 +-------------------------------------- test/Makefile | 10 ++--- 5 files changed, 22 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index d017230..1c289d7 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,9 @@ to generate multi-lingual curricula documentation tables from structured representations as a flatfile database. Data scheme and actual values are kept in YAML files in order to version them with git. -# Usage +## Usage ```sh -$> python coursebuilder usage: [-h] [-m META [META ...]] [-l LANG] [-f FIELDS [FIELDS ...]] [-s SCHEMA] [-q QUERY] [-p] [--title TITLE] [-b BOOK] [--level LEVEL] [--table-gen TABLE_GEN] [--template TEMPLATE] [-o OUT] [--legacy] [--leftcol LEFTCOL] @@ -37,11 +36,11 @@ options: --leftcol LEFTCOL maximum size of left column ``` -# Author +## Author © Copyright 2020-2024 Hartmut Seichter -# Licence +## Licence Coursebuilder is licensed under the terms of the MIT License. For details consult https://opensource.org/license/mit/ or the attached license file diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index b55e6e5..74f7808 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -66,7 +66,12 @@ class CourseBuilder: add_pagebreak=args.pagebreak, title_template=args.title, first_colwidth=args.leftcol) - + elif query: + print(schema.to_list_of_tuple( + meta=yaml.load(fm,Loader=yaml.Loader), + fields=actual_fields, + lang=args.lang)) + pass else: MarkdownGenerator.generate_table( table_items=schema.to_list_of_tuple( @@ -84,6 +89,7 @@ class CourseBuilder: # arguments parser = ArgumentParser(description='versatile curricula generator') + # parameters parser.add_argument('-m','--meta',action="extend", nargs="+", type=str,help="course description(s) as YAML file(s)") parser.add_argument('-l','--lang',help="Language to parse from meta file (use de or en)",default='de') parser.add_argument('-f','--fields',help="Fields to be used, the table will be build accordingly",action="extend", nargs="+", type=str) @@ -132,6 +138,7 @@ class CourseBuilder: if 'text' in section: print(section['text'][args.lang]) + # gernerate section wise parts if 'modules' in section: # override fields @@ -148,5 +155,6 @@ class CourseBuilder: else: parser.print_help() +# run as main if __name__ == '__main__': CourseBuilder.run() diff --git a/coursebuilder/query.py b/coursebuilder/query.py index 835fe36..0a5bb41 100644 --- a/coursebuilder/query.py +++ b/coursebuilder/query.py @@ -2,6 +2,10 @@ import pandas as pd class Query: + """ + Runs pandas.Dataframe.query() with special additions we need + for generating tables for Curricula + """ def __init__(self,query) -> None: self.__query = query diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index 1b0291d..45eb7e1 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -7,12 +7,6 @@ class Schema: def keys(self): return self.__schema.keys() - - # def get_template(self,field,lang='de'): - # if 'template' in self.__schema[field]: - # return self.__schema[field]['template'][lang] - # else: - # return "$value" def is_translatable(self,field): if 'translatable' in self.__schema[field]: @@ -25,94 +19,7 @@ class Schema: return self.__schema[field] else: return False - - # def process_label(self,field,lang='de'): - # # processes the label of a field item - # return self.__schema[field]['label'][lang] - - # def process_str(self,meta,field,lang='de'): - # if self.is_translatable(field): - # return [self.process_label(field,lang),meta[field][lang]] - # else: - # if not 'value' in meta[field]: - # raise AssertionError(field,'incomplete') - - # return [self.process_label(field,lang),meta[field]['value']] - - # def process_enum(self,meta,field,lang='de'): - # """ - # enum have a specification 'specs' option - # that can be forced by the scheme - # """ - # vv = meta[field]['value'] - # enum_val = self.__schema[field]['values'][vv][lang] - - # if self.needs_spec(field): - - # t = string.Template(self.get_template(field=field,lang=lang)) - - # spec = meta[field]['spec'][lang] - - # return [self.process_label(field,lang),t.substitute({'value': enum_val,'spec': spec})] - # else: - # return [self.process_label(field,lang),enum_val] - - # def process_num(self,meta,field,lang='de'): - # v = meta[field]['value'] - # t = string.Template(self.get_template(field,lang)) - # return [self.process_label(field,lang),t.substitute({'value' : v})] - - - # def process_multinum(self,meta,field,lang='de'): - # """multinums have various values""" - # v = meta[field]['value'] - # t = string.Template(self.get_template(field,lang)) - # if hasattr(v, "__len__"): - # vv = [t.substitute({'value' : ev}) for ev in v] - # return [self.process_label(field,lang),', '.join(vv)] - # else: - # return self.process_num(meta=meta,field=field,lang=lang) - - - # def process_multikey(self,meta,field,lang='de'): - # """ - # multikey need to assign a numeric value to a key - # """ - # vs = meta[field]['value'] - # t = string.Template(self.get_template(field,lang)) - - # k = self.process_label(field,lang) - - # parts = [] - - # for e in vs: - # kk = self.__schema[field]['keys'][e][lang] - # parts.append(t.substitute({'key': kk, 'value' : vs[e]})) - - # return [k,', '.join(parts)] - - - # def process(self,meta,fields,lang): - - # table_items = [] - - # # iterate over requested fields - # for field in fields: - # try: - # # correlate with schema and append - # match self.__schema[field]['type']: - # case 'str': table_items.append(self.process_str(meta,field,lang)) - # case 'enum': table_items.append(self.process_enum(meta,field,lang)) - # case 'int' | 'num' : table_items.append(self.process_num(meta,field,lang)) - # case 'multinum' : table_items.append(self.process_multinum(meta,field,lang)) - # case 'multikey': table_items.append(self.process_multikey(meta,field,lang)) - # case _: raise ValueError - # except Exception as exp: - # print(field,' not resolvable in ',self.__schema,exp) - - # # maybe return tableitems as np.Dataframe? - # return table_items - + def get_value(self,meta,field,lang): """ treats receiving the value like a variant, diff --git a/test/Makefile b/test/Makefile index 646a326..b864984 100644 --- a/test/Makefile +++ b/test/Makefile @@ -12,17 +12,17 @@ coursebuilder := ../coursebuilder all: ${targets} -${target_en}: +${target_en}: mod.cg.yaml @echo "creating English version ..." mkdir -p ${build_dir} - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l en -f fields.yaml | pandoc ${target_flags} -o ${target_en} + python ${coursebuilder} -s schema.yaml -m $^ -l en -f fields.yaml | pandoc ${target_flags} -o ${target_en} -${target_de}: +${target_de}: mod.cg.yaml @echo "creating German version ..." mkdir -p ${build_dir} - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de} + python ${coursebuilder} -s schema.yaml -m $^ -l de -f fields.yaml | pandoc ${target_flags} -o ${target_de} -${target_de_book}: +${target_de_book}: *.yaml python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 25 --legacy | pandoc ${target_flags} -V toc:true -V lang:de -o ${target_de_book} clean: From c64b2c20446b2158507e6c96a10b328515553760 Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Mon, 27 May 2024 21:09:36 +0200 Subject: [PATCH 15/19] MVP creating summary tables --- README.md | 10 ++++-- TODO.md | 3 +- coursebuilder/__main__.py | 70 ++++++++++++++++++++++++++++++++------- coursebuilder/query.py | 20 ----------- coursebuilder/schema.py | 11 +++++- test/Makefile | 5 ++- 6 files changed, 82 insertions(+), 37 deletions(-) delete mode 100644 coursebuilder/query.py diff --git a/README.md b/README.md index 1c289d7..732cb5c 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ actual values are kept in YAML files in order to version them with git. ## Usage ```sh -usage: [-h] [-m META [META ...]] [-l LANG] [-f FIELDS [FIELDS ...]] [-s SCHEMA] [-q QUERY] [-p] [--title TITLE] [-b BOOK] [--level LEVEL] [--table-gen TABLE_GEN] - [--template TEMPLATE] [-o OUT] [--legacy] [--leftcol LEFTCOL] +usage: [-h] [-m META [META ...]] [-l LANG] [-f FIELDS [FIELDS ...]] [-s SCHEMA] [-q QUERY] [-qs QUERY_SORT] [-qc QUERY_COMPOUND] [-qf QUERY_FILTER [QUERY_FILTER ...]] + [-p] [--title TITLE] [-b BOOK] [--level LEVEL] [--table-gen TABLE_GEN] [--template TEMPLATE] [-o OUT] [--legacy] [--leftcol LEFTCOL] versatile curricula generator @@ -24,6 +24,12 @@ options: using provided schema -q QUERY, --query QUERY compound query to select items + -qs QUERY_SORT, --query-sort QUERY_SORT + sort query with a min/max over a column like min:credits + -qc QUERY_COMPOUND, --query-compound QUERY_COMPOUND + create a compound from a column with multiple values/dictionaries in cells + -qf QUERY_FILTER [QUERY_FILTER ...], --query-filter QUERY_FILTER [QUERY_FILTER ...] + filter final list of columns for output -p, --pagebreak add a pagebreak after each module --title TITLE template for title - use curly brackets (i.e. {}) to mark where the title string is inserted -b BOOK, --book BOOK process a whole curriculum book with sections diff --git a/TODO.md b/TODO.md index c35be4a..190ad1c 100644 --- a/TODO.md +++ b/TODO.md @@ -6,4 +6,5 @@ * [x] add a book mode for mixing input and headers (# Blah -m mod.cg.yaml) * [~] table generator * [ ] overlay of compulsory with other modes ... -* [ ] add template based generator \ No newline at end of file +* [ ] add template based generator +* [ ] port over to structured YAML ... https://tolgee.io/platform/formats/structured_yaml \ No newline at end of file diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 74f7808..9c88513 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -19,7 +19,6 @@ from tablegenerator import TableGenerator from markdowngenerator import MarkdownGenerator from templategenerator import TemplateGenerator from schema import Schema -from query import Query class CourseBuilder: @@ -47,10 +46,8 @@ class CourseBuilder: if actual_fields == None: actual_fields = list(schema.keys()) - # in case we are running query mode - query = Query(args.query) if args.query else None - result = [] + result_df = [] # iterate through meta files for m in args.meta: @@ -66,12 +63,14 @@ class CourseBuilder: add_pagebreak=args.pagebreak, title_template=args.title, first_colwidth=args.leftcol) - elif query: - print(schema.to_list_of_tuple( + elif args.query: + + lot = schema.to_short_dict( meta=yaml.load(fm,Loader=yaml.Loader), fields=actual_fields, - lang=args.lang)) - pass + lang=args.lang) + + result_df.append(pd.DataFrame([lot])) else: MarkdownGenerator.generate_table( table_items=schema.to_list_of_tuple( @@ -82,6 +81,43 @@ class CourseBuilder: title_template=args.title, first_colwidth=args.leftcol) + # query mode + if args.query and len(result_df): + + # got the list + df = pd.concat(result_df,ignore_index=True) + + # generate a dataframe + df_q = df.query(args.query) + + # generate a compound column --query-compound column:sum + if args.query_compound: + df_q.loc[:,'form-of-instruction.sum'] = df_q['form-of-instruction'].apply(lambda x: sum(list(x.values()))) + + # --query-sort is parameterized as min:credits - hence direction:column + if args.query_sort: + qs = args.query_sort.split(':') + match qs[0]: + case 'min' : df_q = df_q.sort_values(by=qs[1],ascending=True,key=lambda col: min(col) if hasattr(col,'__len()__') else col) + case 'max' : df_q = df_q.sort_values(by=qs[1],ascending=False,key=lambda col: max(col) if hasattr(col,'__len()__') else col) + + # filter query + if args.query_filter: + df_q = df_q.loc[:,args.query_filter] + + # print(df_q.head()) + + q_as_md = df_q.to_markdown(tablefmt='grid',index=False) + + print(q_as_md) + + # # lets get crazy to create a summary table! + # df_summary = pd.DataFrame([{ + # 'sum.credits': df_q['credits'].sum() + # }]) + + # print(df_summary.to_markdown(tablefmt='grid',index=False)) + @staticmethod def run(): @@ -89,13 +125,20 @@ class CourseBuilder: # arguments parser = ArgumentParser(description='versatile curricula generator') - # parameters + # loading mode for internal database parser.add_argument('-m','--meta',action="extend", nargs="+", type=str,help="course description(s) as YAML file(s)") parser.add_argument('-l','--lang',help="Language to parse from meta file (use de or en)",default='de') parser.add_argument('-f','--fields',help="Fields to be used, the table will be build accordingly",action="extend", nargs="+", type=str) - parser.add_argument('-s','--schema',help="using provided schema") - parser.add_argument('-q','--query',help="compound query to select items") - + parser.add_argument('-s','--schema', help="using provided schema") + + # query mode + parser.add_argument('-q','--query', type=str, default=None, help="compound query to select items") + parser.add_argument('-qs','--query-sort',type=str,default=None,help="sort query with a min/max over a column like min:credits") + parser.add_argument('-qc','--query-compound',type=str,default=None,help="create a compound from a column with multiple values/dictionaries in cells") + parser.add_argument('-qf','--query-filter',type=str,default=[],action="extend", nargs="+",help="filter final list of columns for output") + + + # create pagebreaks parser.add_argument('-p','--pagebreak',action="store_true",help="add a pagebreak after each module") parser.add_argument('--title',type=str,default=None,help="template for title - use curly brackets (i.e. {}) to mark where the title string is inserted") parser.add_argument('-b','--book',type=str,help="process a whole curriculum book with sections") @@ -157,4 +200,7 @@ class CourseBuilder: # run as main if __name__ == '__main__': + # recommended setting for pandas + pd.options.mode.copy_on_write = True + # run CourseBuilder.run() diff --git a/coursebuilder/query.py b/coursebuilder/query.py deleted file mode 100644 index 0a5bb41..0000000 --- a/coursebuilder/query.py +++ /dev/null @@ -1,20 +0,0 @@ - -import pandas as pd - -class Query: - """ - Runs pandas.Dataframe.query() with special additions we need - for generating tables for Curricula - """ - - def __init__(self,query) -> None: - self.__query = query - - def run(self,table_items): - # print(table_items) - # for item in table_items: - # pass - # print(item) - # print(eval(self.__query,locals())) - pass - diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index 45eb7e1..c03a47f 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -28,7 +28,7 @@ class Schema: match self.__schema[field]['type']: case 'str': return meta[field][lang] if self.is_translatable(field) else meta[field]['value'] case 'enum' | 'int' | 'num' | 'multikey' : return meta[field]['value'] - case 'multinum': return meta[field]['value'] if hasattr(meta[field]['value'],'__iter__') else (meta[field]['value'],) # force list! + case 'multinum': return meta[field]['value'] if hasattr(meta[field]['value'],'__iter__') else [meta[field]['value'],] # force list! def to_list_of_dict(self,meta,fields,lang): """ @@ -49,6 +49,15 @@ class Schema: } for field in fields] + + def to_short_dict(self,meta,fields,lang): + """ + generates a short version of dict which can easily be converted + to a pandas dataframe + """ + # dict comprehension for whole meta part + return { field : self.get_value(meta,field,lang) for field in fields } + def to_list_of_tuple(self,meta,fields,lang): """ generates a list of tuples with a label and value (text) diff --git a/test/Makefile b/test/Makefile index b864984..d3a5e27 100644 --- a/test/Makefile +++ b/test/Makefile @@ -33,6 +33,9 @@ debug: # | pandoc ${target_flags} -V lang:de -o ${target_de} debug-query: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "item['field'] == 'kind' and item['value'] == 'elective'" + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name id credits + +debug-query-book: + python ${coursebuilder} -s schema.yaml -b book.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name id credits .PHONY: clean \ No newline at end of file From 4ed9804405bf42a8beb8f4f8e1994d5bf1ef719c Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Tue, 28 May 2024 08:03:52 +0200 Subject: [PATCH 16/19] bring back templating --- coursebuilder/__main__.py | 20 ++++++++++++++++++++ coursebuilder/schema.py | 3 +++ test/Makefile | 4 ++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 9c88513..f9fc629 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -14,12 +14,14 @@ from argparse import ArgumentParser import os,sys import yaml import pandas as pd +from string import Template from tablegenerator import TableGenerator from markdowngenerator import MarkdownGenerator from templategenerator import TemplateGenerator from schema import Schema + class CourseBuilder: @staticmethod @@ -107,8 +109,24 @@ class CourseBuilder: # print(df_q.head()) + # set value transforms + if args.query_template: + + ww = 'written' + #df_q['form-of-exam'] = 'Schriftlich' if df_q.loc[:,'form-of-exam'] == 'written' else 'was anderes' + # mm = Template("{'written':'S','oral':'mündlich'}[${v}]")? + # print(mm.format(v=mm)) + + # set labels + if args.query_labels: + df_q.columns = args.query_labels + q_as_md = df_q.to_markdown(tablefmt='grid',index=False) + + + + print(q_as_md) # # lets get crazy to create a summary table! @@ -136,6 +154,8 @@ class CourseBuilder: parser.add_argument('-qs','--query-sort',type=str,default=None,help="sort query with a min/max over a column like min:credits") parser.add_argument('-qc','--query-compound',type=str,default=None,help="create a compound from a column with multiple values/dictionaries in cells") parser.add_argument('-qf','--query-filter',type=str,default=[],action="extend", nargs="+",help="filter final list of columns for output") + parser.add_argument('-ql','--query-labels',type=str,default=[],action="extend", nargs="+",help="new labels for query like") + parser.add_argument('-qt','--query-template',type=str,default=[],action="extend", nargs="+",help="templates for values in the form of {value}") # create pagebreaks diff --git a/coursebuilder/schema.py b/coursebuilder/schema.py index c03a47f..9dfeda5 100644 --- a/coursebuilder/schema.py +++ b/coursebuilder/schema.py @@ -5,6 +5,9 @@ class Schema: def __init__(self,schema) -> None: self.__schema = schema + def __getitem__(self, field): + return self.__schema[field] + def keys(self): return self.__schema.keys() diff --git a/test/Makefile b/test/Makefile index d3a5e27..9fed28a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -33,9 +33,9 @@ debug: # | pandoc ${target_flags} -V lang:de -o ${target_de} debug-query: - python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name id credits + python ${coursebuilder} -s schema.yaml -m mod.cg.yaml mod.interactsys.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name credits form-of-exam -ql Modulname Kreditpunkte Prüfungsart -qt quatsch debug-query-book: - python ${coursebuilder} -s schema.yaml -b book.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name id credits + python ${coursebuilder} -s schema.yaml -b book.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name credits form-of-instruction -ql Modulname Kürzel Kreditpunkte .PHONY: clean \ No newline at end of file From ef011cda557a6ce1656ec4a851f2515fac04d01d Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Tue, 28 May 2024 09:05:20 +0200 Subject: [PATCH 17/19] need to find CLI for generating summary tables --- coursebuilder/__main__.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index f9fc629..f731492 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -117,24 +117,20 @@ class CourseBuilder: # mm = Template("{'written':'S','oral':'mündlich'}[${v}]")? # print(mm.format(v=mm)) + # lets get crazy to create a summary table! + df_summary = pd.DataFrame([{ + 'sum.credits': df_q['credits'].sum() + }]) + # set labels if args.query_labels: df_q.columns = args.query_labels q_as_md = df_q.to_markdown(tablefmt='grid',index=False) - - - - print(q_as_md) - # # lets get crazy to create a summary table! - # df_summary = pd.DataFrame([{ - # 'sum.credits': df_q['credits'].sum() - # }]) - - # print(df_summary.to_markdown(tablefmt='grid',index=False)) + print(df_summary.to_markdown(tablefmt='grid',index=False)) @staticmethod From d41712e01037dc7edf60bfc8e39ddcc4036ec37c Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Wed, 29 May 2024 20:02:48 +0200 Subject: [PATCH 18/19] small update to get better query mode working --- coursebuilder/__main__.py | 56 +++---- docs/quickstart.md | 18 +-- test/Makefile | 6 +- test/schema.yaml | 318 +++++++++++++++----------------------- 4 files changed, 164 insertions(+), 234 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index f731492..9b69f07 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -1,11 +1,11 @@ #!/usr/bin/env python """ -CourseBuilder +CourseBuilder -Coursebuilder is a preprocessor tool for [pandoc](https://pandoc.org) -to generate multi-lingual curricula documentation tables from -structured representations as a flatfile database. Data scheme and +Coursebuilder is a preprocessor tool for [pandoc](https://pandoc.org) +to generate multi-lingual curricula documentation tables from +structured representations as a flatfile database. Data scheme and actual values are kept in YAML files in order to version them with git. """ @@ -27,7 +27,7 @@ class CourseBuilder: @staticmethod def generate(args): if args.schema and args.meta: - + # get actual fields actual_fields = None @@ -37,8 +37,8 @@ class CourseBuilder: actual_fields = yaml.load(ff,Loader=yaml.Loader)['fields'] else: # seem we have a list or None - actual_fields = args.fields - + actual_fields = args.fields + # get schema schema = None with open(args.schema) as f: @@ -66,12 +66,12 @@ class CourseBuilder: title_template=args.title, first_colwidth=args.leftcol) elif args.query: - + lot = schema.to_short_dict( meta=yaml.load(fm,Loader=yaml.Loader), fields=actual_fields, lang=args.lang) - + result_df.append(pd.DataFrame([lot])) else: MarkdownGenerator.generate_table( @@ -85,16 +85,17 @@ class CourseBuilder: # query mode if args.query and len(result_df): - + # got the list df = pd.concat(result_df,ignore_index=True) - + # generate a dataframe df_q = df.query(args.query) # generate a compound column --query-compound column:sum if args.query_compound: - df_q.loc[:,'form-of-instruction.sum'] = df_q['form-of-instruction'].apply(lambda x: sum(list(x.values()))) + print(args.query_compound) + df_q.loc[:,'form-of-instruction.sum'] = df_q['form-of-instruction'].apply(lambda x: sum(list(x.values()))) # --query-sort is parameterized as min:credits - hence direction:column if args.query_sort: @@ -111,7 +112,8 @@ class CourseBuilder: # set value transforms if args.query_template: - + + # no idea yet how to parameterize this ww = 'written' #df_q['form-of-exam'] = 'Schriftlich' if df_q.loc[:,'form-of-exam'] == 'written' else 'was anderes' # mm = Template("{'written':'S','oral':'mündlich'}[${v}]")? @@ -121,8 +123,8 @@ class CourseBuilder: df_summary = pd.DataFrame([{ 'sum.credits': df_q['credits'].sum() }]) - - # set labels + + # set labels directly! if args.query_labels: df_q.columns = args.query_labels @@ -135,7 +137,7 @@ class CourseBuilder: @staticmethod def run(): - + # arguments parser = ArgumentParser(description='versatile curricula generator') @@ -144,7 +146,7 @@ class CourseBuilder: parser.add_argument('-l','--lang',help="Language to parse from meta file (use de or en)",default='de') parser.add_argument('-f','--fields',help="Fields to be used, the table will be build accordingly",action="extend", nargs="+", type=str) parser.add_argument('-s','--schema', help="using provided schema") - + # query mode parser.add_argument('-q','--query', type=str, default=None, help="compound query to select items") parser.add_argument('-qs','--query-sort',type=str,default=None,help="sort query with a min/max over a column like min:credits") @@ -152,8 +154,8 @@ class CourseBuilder: parser.add_argument('-qf','--query-filter',type=str,default=[],action="extend", nargs="+",help="filter final list of columns for output") parser.add_argument('-ql','--query-labels',type=str,default=[],action="extend", nargs="+",help="new labels for query like") parser.add_argument('-qt','--query-template',type=str,default=[],action="extend", nargs="+",help="templates for values in the form of {value}") - - + + # create pagebreaks parser.add_argument('-p','--pagebreak',action="store_true",help="add a pagebreak after each module") parser.add_argument('--title',type=str,default=None,help="template for title - use curly brackets (i.e. {}) to mark where the title string is inserted") @@ -163,22 +165,22 @@ class CourseBuilder: parser.add_argument('--template',type=str,default=None,help='defines a template to be used with fields') parser.add_argument('-o','--out',type=str,default=None,help='set the output type') parser.add_argument('--legacy',action="store_true",help="use legacy generator mode for compatibility") - + parser.add_argument('--leftcol',type=int,default=35,help='maximum size of left column') # get arguments args = parser.parse_args() - + if args.table_gen: - tg = TableGenerator() + tg = TableGenerator() tg.generate_table(args.table_gen) return # book mode with predefined setting from a book file if args.book and args.schema: - + with open(args.book) as bf: actual_fields = [] @@ -199,13 +201,13 @@ class CourseBuilder: # gernerate section wise parts if 'modules' in section: - + # override fields args.fields = actual_fields - + # expand filenames to be relative to the book args.meta = [os.path.join(os.path.dirname(book_path),mod_path) for mod_path in section['modules']] - + CourseBuilder.generate(args=args) # verbose command line mode @@ -217,6 +219,6 @@ class CourseBuilder: # run as main if __name__ == '__main__': # recommended setting for pandas - pd.options.mode.copy_on_write = True + pd.options.mode.copy_on_write = True # run CourseBuilder.run() diff --git a/docs/quickstart.md b/docs/quickstart.md index a293ef4..d682b75 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,23 +1,23 @@ -# Concept +# concept -The concept behind coursebuilder is to store curricula descriptions in `YAML` files that can be versioned in a git repository. Unlike classic databases an observable and well defined versioning is paramount in these descriptions as they are the legal foundation for study and exam regulations. +The concept behind coursebuilder is to store curricula descriptions in `YAML` files that can be versioned in a git repository. Unlike classic databases, an observable and well defined versioning is paramount in these descriptions as they are the legal foundation for study and exam regulations. The following pieces play together here: - `schema` files, usually a `schema.yaml` -- `mod` files, usually something along the lines of `mod.coursecode.yaml` +- `mod` files, usually something along the lines of `mod.coursecode.yaml` - `book` files describing a whole regulation set and course global details -- some sort of transformation with `coursebuilder` into Markdown that is piped through [pandoc](https://pandoc.org) in order to generate PDF, HTML and other representation from this code +- some sort of transformation with `coursebuilder` into Markdown that is piped through [pandoc](https://pandoc.org) in order to generate PDF, HTML and other representation from this code -# schema files +# schema files Schema files are responsible to describe the used structures in a database. The following datatypes are supported: - `str` a simple string, can be accompanied with a `template` - `enum` a classic enum datatype with a fixed set of values -- `num` a numeric datatype +- `num` a numeric datatype - `multinum` an array type with the possibility to `spec` each value -- `multikey` a key-value type with additional numeric data associated with each key instance +- `multikey` a key-value type with additional numeric data associated with each key instance # mod files (modules) @@ -31,11 +31,11 @@ Modules describe a course in detail and implement an instance of the schema file ```yaml # this would reside in a schema field on top level # a field of name 'id' -id: # name of the field +id: # name of the field type: str # sets the datatype to str translatable: false # enforces the value is not translatable (default is true) label: { # label describes the meaning of the datatype in regards of the schema de: "Kürzel", # translation of the label in German (de) en: "code" # translation of the label in English (en) } -``` \ No newline at end of file +``` diff --git a/test/Makefile b/test/Makefile index 9fed28a..6aa2e73 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,4 +1,4 @@ - +# debug make file for testing build_dir := build target_en := ${build_dir}/table.en.pdf target_de := ${build_dir}/table.de.pdf @@ -25,7 +25,7 @@ ${target_de}: mod.cg.yaml ${target_de_book}: *.yaml python ${coursebuilder} -s schema.yaml -b book.yaml -p --title "### {}" -l de --leftcol 25 --legacy | pandoc ${target_flags} -V toc:true -V lang:de -o ${target_de_book} -clean: +clean: rm -f ${targets} debug: @@ -38,4 +38,4 @@ debug-query: debug-query-book: python ${coursebuilder} -s schema.yaml -b book.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name credits form-of-instruction -ql Modulname Kürzel Kreditpunkte -.PHONY: clean \ No newline at end of file +.PHONY: clean diff --git a/test/schema.yaml b/test/schema.yaml index a7cf2f3..fac24c4 100644 --- a/test/schema.yaml +++ b/test/schema.yaml @@ -1,300 +1,228 @@ # fields in curricular description -# leaning on methods in OpenAPI 3.0 +# leaning on methods in OpenAPI 3.0 # # Modulname # name: - type: str - label: - de: "Modulname" - en: "name of course" + type: str + label: + de: "Modulname" + en: "name of course" # # Modulverantwortliche:r # -instructor: - type: str - label: - de: "Modulverantwortlicher / Modulverantwortliche" - en: "module instructor" - +instructor: + type: str + label: + de: "Modulverantwortlicher / Modulverantwortliche" + en: "module instructor" # # Kürzel / ID # -id: - type: str - translatable: false - label: { - de: "Kürzel", - en: "code" - } - +id: + type: str + translatable: false + label: { de: "Kürzel", en: "code" } # # Qualifikationsziele # -# Welche fachbezogenen, methodischen, fachübergreifende Kompetenzen, -# Schlüsselqualifikationen - werden erzielt (erworben)? Diese sind +# Welche fachbezogenen, methodischen, fachübergreifende Kompetenzen, +# Schlüsselqualifikationen - werden erzielt (erworben)? Diese sind # an der zu definierenden Gesamtqualifikation (angestrebter Abschluss) auszurichten. -# +# # Lernergebnisse sind Aussagen darüber, was ein Studierender nach Abschluss des Moduls weiß, -# versteht und in der Lage ist zu tun. Die Formulierung sollte sich am Qualifikationsrahmen +# versteht und in der Lage ist zu tun. Die Formulierung sollte sich am Qualifikationsrahmen # für Deutsche Hochschulabschlüsse orientieren und Inhaltswiederholungen vermeiden. -# -# Des Weiteren finden Sie im QM-Portal die „Handreichung zur Beschreibung von Lernzielen“ +# +# Des Weiteren finden Sie im QM-Portal die „Handreichung zur Beschreibung von Lernzielen“ # als Formulierungshilfe. goal: - type: str - label: { - de: "Qualifikationsziele", - en: "educational goal" - } + type: str + label: { de: "Qualifikationsziele", en: "educational goal" } # # Modulinhalte # -# Welche fachlichen, methodischen, fachpraktischen und fächerübergreifenden -# Inhalte sollen vermittelt werden? -# +# Welche fachlichen, methodischen, fachpraktischen und fächerübergreifenden +# Inhalte sollen vermittelt werden? +# # Es ist ein stichpunktartiges Inhaltsverzeichnis zu erstellen. -content: - type: str - label: { - de: "Modulinhalte", - en: "content" - } +content: + type: str + label: { de: "Modulinhalte", en: "content" } # # Lehrform # # -# Welche Lehr- und Lernformen werden angewendet? -# (Vorlesungen, Übungen, Seminare, Praktika, +# Welche Lehr- und Lernformen werden angewendet? +# (Vorlesungen, Übungen, Seminare, Praktika, # Projektarbeit, Selbststudium) # # Es sind nur Werte aus der Prüfungsordung zugelassen -# -form-of-instruction: - type: multikey - label: { - de: "Lehrform(en)", - en: "form of instruction" +# +form-of-instruction: + type: multikey + label: { de: "Lehrform(en)", en: "form of instruction" } + keys: + { + "lecture": { de: "Vorlesung", en: "lecture" }, + "lecture_seminar": + { de: "Seminaristische Vorlesung", en: "lecture and seminar" }, + "seminar": { de: "Seminar", en: "seminar" }, + "exersise": { de: "Übung", en: "lab exersise" }, + "pc_lab": { de: "Rechnergestütztes Praktikum", en: "PC exersise" }, + "project": { de: "Project", en: "project" }, } - keys: { - 'lecture' : { - de: "Vorlesung", - en: "lecture" - }, - 'lecture_seminar' : { - de: "Seminaristische Vorlesung", - en: "lecture and seminar" - }, - 'seminar' : { - de: "Seminar", - en: "seminar" - }, - 'exersise' : { - de: "Übung", - en: "lab exersise" - }, - 'pc_lab' : { - de: "Rechnergestütztes Praktikum", - en: "PC exersise" - }, - 'project' : { - de: "Project", - en: "project" - } - } - template: - de: "{key} ({value}SWS)" - en: "{key} ({value}SWS)" + template: + de: "{key} ({value}SWS)" + en: "{key} ({value}SWS)" # # Voraussetzungen für die Teilnahme # -# Für jedes Modul sind die Voraussetzungen für die Teilnahme zu beschreiben. +# Für jedes Modul sind die Voraussetzungen für die Teilnahme zu beschreiben. # Welche Kenntnisse, Fähigkeiten und Fertigkeiten sind für eine # erfolgreiche Teilnahme vorauszusetzen? -# -# Alternativ können die Module benannt werden welche für die erfolgreiche +# +# Alternativ können die Module benannt werden welche für die erfolgreiche # Teilnahme im Vorfeld zu belegen sind. -prerequisites: - type: str - label: { - de: "Voraussetzungen für die Teilnahme", - en: "prerequisites" - } +prerequisites: + type: str + label: { de: "Voraussetzungen für die Teilnahme", en: "prerequisites" } # # Literatur und multimediale Lehr- und Lernprogramme # -# +# # Wie können die Studierenden sich auf die Teilnahme an diesem Modul vorbereiten? -# -teaching-material: - type: str - label: { - de: "Literatur und multimediale Lehr- und Lernprogramme", - en: "media of instruction" +# +teaching-material: + type: str + label: + { + de: "Literatur und multimediale Lehr- und Lernprogramme", + en: "media of instruction", } # # Lehrbriefautor # -author-of-indenture: - type: str - label: { - de: "Lehrbriefautor", - en: "author of indenture" - } +author-of-indenture: + type: str + label: { de: "Lehrbriefautor", en: "author of indenture" } # # Verwendung in (Studienprogramm) # used-in: - type: str - label: { - de: "Verwendung", - en: "used in study programs" - } + type: str + label: { de: "Verwendung", en: "used in study programs" } # # Arbeitsaufwand # workload: - type: str - label: { - de: "Arbeitsaufwand / Gesamtworkload", - en: "workload" - } + type: str + label: { de: "Arbeitsaufwand / Gesamtworkload", en: "workload" } # # credits/ECTS # credits: - type: num - unit: ECTS - label: { - en: "credits and weight of mark", - de: "Kreditpunkte und Gewichtung der Note in der Gesamtnote" + type: num + unit: ECTS + label: + { + en: "credits and weight of mark", + de: "Kreditpunkte und Gewichtung der Note in der Gesamtnote", } - template: - de: "{value}CP, Gewichtung: {value}CP von 120CP " - en: "{value}CP, weight: {value} / 120 " - + template: + de: "{value}CP, Gewichtung: {value}CP von 120CP " + en: "{value}CP, weight: {value} / 120 " # # Leistungsnachweis # -form-of-exam: - type: enum - label: { - de: "Leistungsnachweis", - en: "form of examination" +form-of-exam: + type: enum + label: { de: "Leistungsnachweis", en: "form of examination" } + values: + { + "written": { de: "Schriftliche Prüfung", en: "written exam" }, + "oral": { de: "Mündliche Prüfung", en: "oral exam" }, + "alternative": + { de: "Alternative Prüfungunsleistung", en: "alternative examination" }, } - values: { - 'written' : { - de: "Schriftliche Prüfung", - en: "written exam" - }, - 'oral' : { - de: "Mündliche Prüfung", - en: "oral exam" - }, - 'alternative' : { - de: "Alternative Prüfungunsleistung", - en: "alternative examination" - } - } - spec: true - template: - de: "{value} ({spec})" - en: "{value} ({spec})" - + spec: true + template: + de: "{value} ({spec})" + en: "{value} ({spec})" # # Semester # -term: - type: multinum - label: { - de: "Semester", - en: "term" - } - template: - de: "{value}\\. Semester" - en: "{value}\\. semester" +term: + type: multinum + label: { de: "Semester", en: "term" } + template: + de: "{value}\\. Semester" + en: "{value}\\. semester" # # Häufigkeit des Angebots # -frequency: - type: enum - label: { - de: "Häufigkeit des Angebots", - en: "frequency of Offer" - } - values: { - 'once_per_term' : { - de: "jedes Semester", - en: "every semester" - }, - 'once_per_year' : { - de: "einmal im Studienjahr", - en: "once per study year" - } +frequency: + type: enum + label: { de: "Häufigkeit des Angebots", en: "frequency of Offer" } + values: + { + "once_per_term": { de: "jedes Semester", en: "every semester" }, + "once_per_year": + { de: "einmal im Studienjahr", en: "once per study year" }, } # # Dauer des Angebots # -duration: - type: int - label: - de: Dauer - en: duration - template: - de: "{value} Semester" - en: "{value} term(s)" +duration: + type: int + label: + de: Dauer + en: duration + template: + de: "{value} Semester" + en: "{value} term(s)" # # Art der Veranstaltung # kind: - type: enum - label: { - de: 'Art der Veranstaltung (Pflicht, Wahl, etc.)', - en: 'kind of module (compulsory, elective)' + type: enum + label: + { + de: "Art der Veranstaltung (Pflicht, Wahl, etc.)", + en: "kind of module (compulsory, elective)", } - values: { - 'compulsory': { - de: "Pflicht", - en: "compulsory" - }, - 'elective' : { - de: "Wahl/Wahlpflicht", - en: "elective" - } + values: + { + "compulsory": { de: "Pflicht", en: "compulsory" }, + "elective": { de: "Wahl/Wahlpflicht", en: "elective" }, } # # Freiform Bemerkungen # -remarks: - type: str - label: { - de: "Besonderes", - en: "remarks" - } - - - \ No newline at end of file +remarks: + type: str + label: { de: "Besonderes", en: "remarks" } From f45f7b715b99ee3136c606a11f166dce37015d6b Mon Sep 17 00:00:00 2001 From: Hartmut Seichter Date: Mon, 17 Mar 2025 14:37:45 +0100 Subject: [PATCH 19/19] checkint before trying to switch over to datafiles --- coursebuilder/__main__.py | 13 +++++++------ test/Makefile | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/coursebuilder/__main__.py b/coursebuilder/__main__.py index 9b69f07..261672c 100644 --- a/coursebuilder/__main__.py +++ b/coursebuilder/__main__.py @@ -94,8 +94,9 @@ class CourseBuilder: # generate a compound column --query-compound column:sum if args.query_compound: - print(args.query_compound) - df_q.loc[:,'form-of-instruction.sum'] = df_q['form-of-instruction'].apply(lambda x: sum(list(x.values()))) + # print('{}.sum'.format(args.query_compound)) + df_q.loc[:,'{}.sum'.format(args.query_compound)] = df_q[args.query_compound].apply(lambda x: sum(list(x.values()))) + print(df_q) # --query-sort is parameterized as min:credits - hence direction:column if args.query_sort: @@ -120,9 +121,9 @@ class CourseBuilder: # print(mm.format(v=mm)) # lets get crazy to create a summary table! - df_summary = pd.DataFrame([{ - 'sum.credits': df_q['credits'].sum() - }]) + # df_summary = pd.DataFrame([{ + # 'sum.credits': df_q['credits'].sum() + # }]) # set labels directly! if args.query_labels: @@ -132,7 +133,7 @@ class CourseBuilder: print(q_as_md) - print(df_summary.to_markdown(tablefmt='grid',index=False)) + # print(df_summary.to_markdown(tablefmt='grid',index=False)) @staticmethod diff --git a/test/Makefile b/test/Makefile index 6aa2e73..8382633 100644 --- a/test/Makefile +++ b/test/Makefile @@ -38,4 +38,7 @@ debug-query: debug-query-book: python ${coursebuilder} -s schema.yaml -b book.yaml -q "kind=='compulsory'" -qs min:credits -qc form-of-instruction -qf name credits form-of-instruction -ql Modulname Kürzel Kreditpunkte +debug-query-full: + python ${coursebuilder} -s ~/Documents/MaACS/MHB/schema.yaml -b ~/Documents/MaACS/MHB/book.yaml -q "kind=='compulsory_elective'" -qc form-of-instruction -qf name form-of-instruction.sum credits term -ql Modulname SWS Kreditpunkte Semester + .PHONY: clean