#!/usr/bin/env python # Copyright (c) OpenMMLab. All rights reserved. import os import os.path as osp import re from collections import defaultdict from glob import glob from addict import Addict from titlecase import titlecase def _get_model_docs(): """Get all model document files. Returns: list[str]: file paths """ config_root = osp.join('..', '..', 'configs') pattern = osp.sep.join(['*'] * 4) + '.md' docs = glob(osp.join(config_root, pattern)) docs = [doc for doc in docs if '_base_' not in doc] return docs def _parse_model_doc_path(path): """Parse doc file path. Typical path would be like: configs////.md An example is: "configs/animal_2d_keypoint/topdown_heatmap/ animalpose/resnet_animalpose.md" Returns: tuple: - task (str): e.g. ``'Animal 2D Keypoint'`` - dataset (str): e.g. ``'animalpose'`` - keywords (tuple): e.g. ``('topdown heatmap', 'resnet')`` """ _path = path.split(osp.sep) _rel_path = _path[_path.index('configs'):] # get task def _titlecase_callback(word, **kwargs): if word == '2d': return '2D' if word == '3d': return '3D' task = titlecase( _rel_path[1].replace('_', ' '), callback=_titlecase_callback) # get dataset dataset = _rel_path[3] # get keywords keywords_algo = (_rel_path[2], ) keywords_setting = tuple(_rel_path[4][:-3].split('_')) keywords = keywords_algo + keywords_setting return task, dataset, keywords def _get_paper_refs(): """Get all paper references. Returns: Dict[str, List[str]]: keys are paper categories and values are lists of paper paths. """ papers = glob('../src/papers/*/*.md') paper_refs = defaultdict(list) for fn in papers: category = fn.split(osp.sep)[3] paper_refs[category].append(fn) return paper_refs def _parse_paper_ref(fn): """Get paper name and indicator pattern from a paper reference file. Returns: tuple: - paper_name (str) - paper_indicator (str) """ indicator = None with open(fn, 'r', encoding='utf-8') as f: for line in f.readlines(): if line.startswith('', '', indicator).strip() return paper_name, indicator def main(): # Build output folders os.makedirs('model_zoo', exist_ok=True) os.makedirs('model_zoo_papers', exist_ok=True) # Collect all document contents model_doc_list = _get_model_docs() model_docs = Addict() for path in model_doc_list: task, dataset, keywords = _parse_model_doc_path(path) with open(path, 'r', encoding='utf-8') as f: doc = { 'task': task, 'dataset': dataset, 'keywords': keywords, 'path': path, 'content': f.read() } model_docs[task][dataset][keywords] = doc # Write files by task for task, dataset_dict in model_docs.items(): lines = [f'# {task}', ''] for dataset, keywords_dict in dataset_dict.items(): lines += [ '
', '

', '', f'## {titlecase(dataset)} Dataset', '' ] for keywords, doc in keywords_dict.items(): keyword_strs = [ titlecase(x.replace('_', ' ')) for x in keywords ] dataset_str = titlecase(dataset) if dataset_str in keyword_strs: keyword_strs.remove(dataset_str) lines += [ '
', '', (f'### {" + ".join(keyword_strs)}' f' on {dataset_str}'), '', doc['content'], '' ] fn = osp.join('model_zoo', f'{task.replace(" ", "_").lower()}.md') with open(fn, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) # Write files by paper paper_refs = _get_paper_refs() for paper_cat, paper_list in paper_refs.items(): lines = [] for paper_fn in paper_list: paper_name, indicator = _parse_paper_ref(paper_fn) paperlines = [] for task, dataset_dict in model_docs.items(): for dataset, keywords_dict in dataset_dict.items(): for keywords, doc_info in keywords_dict.items(): if indicator not in doc_info['content']: continue keyword_strs = [ titlecase(x.replace('_', ' ')) for x in keywords ] dataset_str = titlecase(dataset) if dataset_str in keyword_strs: keyword_strs.remove(dataset_str) paperlines += [ '
', '', (f'### {" + ".join(keyword_strs)}' f' on {dataset_str}'), '', doc_info['content'], '' ] if paperlines: lines += ['
', '

', '', f'## {paper_name}', ''] lines += paperlines if lines: lines = [f'# {titlecase(paper_cat)}', ''] + lines with open( osp.join('model_zoo_papers', f'{paper_cat.lower()}.md'), 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) if __name__ == '__main__': print('collect model zoo documents') main()