1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
import dataclasses
import datetime
import os
import logging
import pathlib
import glob
import typing
import markdown
import yaml
# Known exceptions, these are raised when generating on program exiting error, and are displayed to the user nicely
class ArticleHandlerException(Exception):
pass
class ArticleMetaDataMalformed(ArticleHandlerException):
pass
class ArticleNoMetaData(ArticleHandlerException):
pass
@dataclasses.dataclass
class WebPage:
url: str
name: str
@dataclasses.dataclass
class MetaData():
description: str
created: datetime.date
@property
def pretty_print(self) -> str:
return f'''
Description: {self.description}
Created: {self.created}'''
def truncate(text: str, max_length: int = 50):
if len(text) < max_length:
return text
else:
return text[0:max_length - 1]
@dataclasses.dataclass
class Article(WebPage):
metadata: MetaData
web_dir: tuple
source_path: str
source: str
html: str
folder_path: typing.Union[None, str] = None
@property
def pretty_print(self) -> str:
return f'''
Name: {self.name}
Metadata: {self.metadata.pretty_print}
Web dir: {self.web_dir}
URL: {self.url}
Source path: {self.source_path}
Folder path: {self.folder_path}
HTML: {truncate(self.html)}
Source: {truncate(self.source)}'''
@dataclasses.dataclass
class Folder(WebPage):
articles: typing.List[Article] = dataclasses.field(default_factory=list)
sub_folders: typing.Dict[str, object] = dataclasses.field(default_factory=dict)
def get_web_dir(path, name) -> tuple[str, ...]:
dir_structure = path.split(name)[0]
# Split into tuple, remove first part, assemble to path again, all to remove the first source dir agnostically
dir_structure_prefix_striped = pathlib.Path(dir_structure).parts[2:]
return dir_structure_prefix_striped
def parse_article_meta_data(source: str) -> typing.Tuple[str, MetaData]:
if source.startswith('---'):
meta_data_yml_end_char_index = source.find('---', 3)
meta_data_yml = source[3:meta_data_yml_end_char_index]
# Strip metadata text from source, before feeding it to the markdown reader
source = source[meta_data_yml_end_char_index + 3:]
meta_data = yaml.safe_load(meta_data_yml)
try:
meta_data = MetaData(**meta_data)
except TypeError as type_error_exception:
raise ArticleMetaDataMalformed(str(type_error_exception))
return source, meta_data
else:
raise ArticleNoMetaData(f'No metadata found')
def get_article(path: str) -> Article:
logging.debug(path)
article_args = {}
filename, file_extension = os.path.splitext(path)
basename = os.path.basename(filename)
if basename == 'index':
# Article is the folder
article_folder_name = os.path.dirname(path)
article_args['folder_path'] = article_folder_name
dir_basename = os.path.basename(article_folder_name)
article_args['name'] = dir_basename
else:
# Article one file
article_args['name'] = os.path.basename(filename)
article_args['web_dir'] = get_web_dir(path, article_args['name'])
article_args['source_path'] = path
article_args[
'url'] = f'{"/" if article_args["web_dir"] else ""}{"/".join(article_args["web_dir"])}/{article_args["name"]}'
with open(path, 'r') as file:
source = file.read()
article_args['source'], article_args['metadata'] = parse_article_meta_data(source)
article_args['html'] = markdown.markdown(article_args['source'], extensions=['fenced_code', 'codehilite', 'tables'])
article = Article(**article_args)
return article
def discover_folder_structure(article: Article, articles: Folder):
previous_folder = articles
for folder_name in article.web_dir:
logging.debug(folder_name)
if folder_name not in previous_folder.sub_folders:
logging.debug('new')
current_folder = Folder(url=f'{previous_folder.url}{folder_name}/', name=folder_name)
previous_folder.sub_folders[folder_name] = current_folder
else:
logging.debug('reuse')
current_folder = previous_folder.sub_folders[folder_name]
previous_folder = current_folder
logging.debug('')
previous_folder.articles.append(article)
def sort_articles(folder: Folder):
folder.articles = sorted(folder.articles,
key=lambda x: x.metadata.created if x.metadata.created is not None else datetime.date.min,
reverse=True)
for folder in folder.sub_folders.values():
sort_articles(folder)
def discover_articles(path):
articles_paths = glob.glob(f'{path}/**/*.md', recursive=True) # Equivalent to ls ./**.md
articles = Folder(url='/', name='index')
for article_path in articles_paths:
article = get_article(article_path)
logging.debug(article.pretty_print)
discover_folder_structure(articles=articles, article=article)
sort_articles(articles)
return articles
|
