diff --git a/.Rbuildignore b/.Rbuildignore index fdd1b8d203..8c099646ca 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -27,7 +27,6 @@ ^Makefile$ ^NEWS\.0\.md$ ^NEWS\.1\.md$ -^_pkgdown\.yml$ ^src/Makevars$ ^CODEOWNERS$ ^GOVERNANCE\.md$ @@ -46,11 +45,11 @@ ^.*\.dll$ ^bus$ -^pkgdown$ ^docs$ ^lib$ ^library$ ^devwd$ +^site$ # only the inst/po compressed files are needed, not raw .pot/.po ^po$ diff --git a/.gitignore b/.gitignore index dba3d1306e..eb5f090fca 100644 --- a/.gitignore +++ b/.gitignore @@ -57,5 +57,10 @@ dev.R *.diff *.patch -# pkgdown -docs +# litedown +site/*.html +site/*.md +site/doc/ +site/man/ +site/reference/ +site/reference_index.rds diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05cc1e445c..3d6fe8dfb8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -351,20 +351,32 @@ test-mac-old: # merging package tarballs and binaries into single R repository # rendering documentation # setting up CRAN-like structure -# generating pkgdown website +# generating litedown website integration: stage: integration - image: registry.gitlab.com/rdatatable/dockerfiles/r-pkgdown + image: registry.gitlab.com/rdatatable/dockerfiles/r-litedown tags: - saas-linux-medium-amd64 - only: - - master + rules: + - if: '$CI_COMMIT_REF_NAME == "master"' + - if: '$CI_PIPELINE_SOURCE == "web"' needs: ["mirror-packages","build","test-lin-rel","test-lin-rel-cran","test-lin-dev-gcc-strict-cran","test-lin-dev-clang-cran","test-lin-rel-vanilla","test-lin-ancient-cran","test-lin-dev-clang-san","test-lin-dev-gcc-san","test-win-rel","test-win-dev" ,"test-win-old","test-mac-rel","test-mac-old"] script: - R --version - - *install-deps ## markdown pkg not present in r-pkgdown image - - mkdir -p ./pkgdown/favicon/ && cp .graphics/favicon/* ./pkgdown/favicon/ ## copy favicons - - Rscript -e 'pkgdown::build_site(override=list(destination="./website"))' + - *install-deps ## markdown pkg not present in r-litedown image + ## Install data.table from build artifacts (needed by litedown site generation) + - R CMD INSTALL bus/build/data.table_*.tar.gz + ## Copy assets to site directory + - mkdir -p ./site/assets/favicon/ + - cp .graphics/favicon/* ./site/assets/favicon/ + ## Generate reference pages first (creates reference_index.rds needed by manual.Rmd) + - Rscript site/generate_reference.R + ## Build litedown site (now manual.Rmd can use reference_index.rds) + - cd site && Rscript -e 'litedown::fuse_site()' && cd .. + - mkdir -p website + - cp -r site/*.html website/ + - cp -r site/reference website/ 2>/dev/null || true + - cp -r site/assets website/ ## html manual, vignettes, repos, cran_web, cran_checks - echo 'source(".ci/ci.R"); source(".ci/publish.R")' >> .Rprofile ## list of available test-* jobs dynamically based on bus/test-* directories @@ -436,8 +448,16 @@ integration: - Rscript -e 'check.index("data.table", names(test.jobs))' ## web/checks/check_flavors.html - Rscript -e 'check.flavors(names(test.jobs))' - ## pkgdown merge - - Rscript -e 'common_files<-function(path1, path2) intersect(list.files(path1, all.files=TRUE, no..=TRUE), list.files(path2, all.files=TRUE, no..=TRUE)); msg = if (length(f<-common_files("website","bus/integration/cran"))) paste(c("Following artifacts will be overwritten by pkgdown artifacts:", paste0(" ", f)), collapse="\n") else "No overlapping files from pkgdown artifacts"; message(msg); q("no")' + ## Add backwards compatibility redirects + - mkdir -p website/news website/reference website/articles + - echo '' > website/news/index.html + - echo '' > website/reference/index.html + ## Render translated vignettes into articles/ + - Rscript site/render_translations.R + ## Copy existing vignettes into articles/ + - cp -r bus/integration/cran/library/data.table/doc/* website/articles/ + ## litedown merge + - Rscript -e 'common_files<-function(path1, path2) intersect(list.files(path1, all.files=TRUE, no..=TRUE), list.files(path2, all.files=TRUE, no..=TRUE)); msg = if (length(f<-common_files("website","bus/integration/cran"))) paste(c("Following artifacts will be overwritten by litedown artifacts:", paste0(" ", f)), collapse="\n") else "No overlapping files from litedown artifacts"; message(msg); q("no")' - mv website/* bus/integration/cran/ ## add plausible.io stats - find bus/integration/cran -type f -iname "*.html" | xargs sed -i 's!!!g' @@ -447,7 +467,6 @@ integration: # R repository # test jobs summaries # html documentation of all packages in repo -# pkgdown website pages: stage: deploy environment: production diff --git a/NEWS.md b/NEWS.md index 10c3e6961b..a5e73e24ef 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -## data.table news and updates +# data.table news and updates **If you are viewing this file on CRAN, please check [latest news on GitHub](https://github.com/Rdatatable/data.table/blob/master/NEWS.md) where the formatting is also better.** diff --git a/_pkgdown.yml b/_pkgdown.yml deleted file mode 100644 index 63e50c248a..0000000000 --- a/_pkgdown.yml +++ /dev/null @@ -1,82 +0,0 @@ -url: https://rdatatable.gitlab.io/data.table - -template: - bootstrap: 5 - light-switch: true - -development: - version_tooltip: "Development version" - -news: - one_page: true - -home: - links: - - text: CRAN-like website - href: web/packages/data.table/index.html - - text: CRAN-like checks - href: web/checks/check_results_data.table.html - - text: Community blog - href: https://rdatatable-community.github.io/The-Raft/ - -navbar: - structure: - left: [home, introduction, articles, news, benchmarks, presentations, communityarticles, reference, cheatsheet] - right: [search, github, lightswitch] - components: - home: - icon: fas fa-home fa-lg - href: index.html - introduction: - text: Introduction - href: articles/datatable-intro.html - articles: - text: Vignettes - menu: - - text: "Introduction to data.table" - href: articles/datatable-intro.html - - text: "Reference semantics" - href: articles/datatable-reference-semantics.html - - text: "Using .SD for Data Analysis" - href: articles/datatable-sd-usage.html - - text: "Keys and fast binary search based subset" - href: articles/datatable-keys-fast-subset.html - - text: "Joins in data.table" - href: articles/datatable-joins.html - - text: "Secondary indices and auto indexing" - href: articles/datatable-secondary-indices-and-auto-indexing.html - - text: "Efficient reshaping using data.table" - href: articles/datatable-reshape.html - - text: "Programming on data.table" - href: articles/datatable-programming.html - - text: "Frequently asked questions" - href: articles/datatable-faq.html - - text: "Importing data.table" - href: articles/datatable-importing.html - - text: "Benchmarking data.table" - href: articles/datatable-benchmarking.html - news: - text: News - href: news/index.html - benchmarks: - text: Benchmarks - href: https://duckdblabs.github.io/db-benchmark - presentations: - text: Presentations - href: https://github.com/Rdatatable/data.table/wiki/Presentations - communityarticles: - text: Articles - href: https://github.com/Rdatatable/data.table/wiki/Articles - reference: - text: Manual - href: reference/index.html - cheatsheet: - text: Cheatsheet - menu: - - text: "English" - href: datatable_cheatsheet.pdf - - text: "Français" - href: datatable_cheatsheet_fr.pdf - github: - icon: fab fa-github fa-lg - href: https://github.com/Rdatatable/data.table diff --git a/site/_litedown.yml b/site/_litedown.yml new file mode 100644 index 0000000000..0261ca07d5 --- /dev/null +++ b/site/_litedown.yml @@ -0,0 +1,19 @@ +--- +site: + rebuild: "always" + pattern: "[.]Rmd$" + +output: + html: + meta: + css2: ["https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css", "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css", "assets/style.css"] + js2: ["https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js"] + title: "data.table" + lang: "en" + footer: "" + include_before: "_navbar.html" + include_after: '' + header_includes: '' + options: + toc: false +--- diff --git a/site/_navbar.html b/site/_navbar.html new file mode 100644 index 0000000000..e1ddaa9b73 --- /dev/null +++ b/site/_navbar.html @@ -0,0 +1,44 @@ + diff --git a/pkgdown/assets/datatable_cheatsheet.pdf b/site/assets/datatable_cheatsheet.pdf similarity index 100% rename from pkgdown/assets/datatable_cheatsheet.pdf rename to site/assets/datatable_cheatsheet.pdf diff --git a/pkgdown/assets/datatable_cheatsheet_fr.pdf b/site/assets/datatable_cheatsheet_fr.pdf similarity index 100% rename from pkgdown/assets/datatable_cheatsheet_fr.pdf rename to site/assets/datatable_cheatsheet_fr.pdf diff --git a/site/assets/style.css b/site/assets/style.css new file mode 100644 index 0000000000..bb361c7b0e --- /dev/null +++ b/site/assets/style.css @@ -0,0 +1,70 @@ +/* data.table website styles */ + +body { + font-family: "Helvetica Neue", Arial, sans-serif; + line-height: 1.6; + color: #212529; + margin: 0 !important; + padding: 0 !important; + max-width: none !important; +} + +/* Layout */ +.frontmatter, +.body { + max-width: 1320px !important; + width: 100% !important; + margin: 0 auto !important; + padding: 0 clamp(1.5rem, 3vw, 4rem) !important; +} + +.frontmatter { + padding-top: 1.5rem; +} + +.frontmatter .title, +.frontmatter .title h1 { + text-align: left; +} + +.external-link::after { + content: "ext"; + font-size: 0.75rem; + margin-left: 0.25rem; +} + +/* Code blocks */ +pre { + background-color: #f8f9fa; + border: 1px solid #dee2e6; + border-radius: 4px; + padding: 1rem; + overflow-x: auto; +} + +code { + background-color: #f8f9fa; + padding: 0.2rem 0.35rem; + border-radius: 3px; + font-family: "SFMono-Regular", Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + font-size: 0.875em; +} + +pre code { + background-color: transparent; + padding: 0; +} + +@media (min-width: 1400px) { + .frontmatter, + .body { + padding: 0 3rem; + } +} + +@media (min-width: 1800px) { + .frontmatter, + .body { + padding: 0 5rem; + } +} diff --git a/site/generate_reference.R b/site/generate_reference.R new file mode 100644 index 0000000000..5b90c41fae --- /dev/null +++ b/site/generate_reference.R @@ -0,0 +1,123 @@ +#!/usr/bin/env Rscript +# Generate individual reference pages from .Rd files + +library(tools) +library(xml2) + +# Create reference directory +ref_dir = "site/reference" +if (!dir.exists(ref_dir)) dir.create(ref_dir, recursive = TRUE) + +# Get all .Rd files +rd_files = list.files("man", pattern = "\\.Rd$", full.names = TRUE) + +# Function to extract metadata from .Rd file +get_rd_info = function(rd_file) { + rd = parse_Rd(rd_file) + + title_idx = which(sapply(rd, attr, "Rd_tag") == "\\title") + title = if (length(title_idx) > 0) { + gsub("^\\s+|\\s+$", "", paste(unlist(rd[[title_idx[1]]]), collapse = "")) + } else NA + + alias_idx = which(sapply(rd, attr, "Rd_tag") == "\\alias") + aliases = if (length(alias_idx) > 0) { + sapply(alias_idx, function(i) paste(unlist(rd[[i]]), collapse = "")) + } else character(0) + + list( + file = basename(rd_file), + name = sub("\\.Rd$", "", basename(rd_file)), + title = title, + aliases = aliases + ) +} + +# Generate HTML for each .Rd file +cat("Generating individual reference pages...\n") + +all_info = lapply(setNames(nm = rd_files), get_rd_info) +topic_links = character() +for (topic in all_info) topic_links[topic$aliases] = paste0('reference/', topic$name, '.html') + +html_meta = xfun::yaml_load(readLines('site/_litedown.yml'))$output$html$meta + +for (rd_file in rd_files) { + info = all_info[[rd_file]] + + out_file = file.path(ref_dir, paste0(info$name, ".html")) + + temp_html = tempfile(fileext = ".html") + Rd2HTML(rd_file, out = temp_html, package = "data.table", Links = topic_links, Links2 = character()) + html_root = read_html(temp_html) + + if (!is.null(html_meta$lang)) + xml_attr(html_root, 'lang') = html_meta$lang + + html_title = xml_find_first(html_root, '/html/head/title') + xml_text(html_title) <- sprintf('%s — %s • data.table', paste(info$aliases, collapse = ", "), info$title) + + html_head = xml_find_first(html_root, '//head') + + if (!is.null(add_head <- html_meta$header_includes)) { + add_head = xml_find_first(read_html(paste0('', add_head, '')), '//head') + for (tag in xml_children(add_head)) + xml_add_child(html_head, tag) + } + + for (css in html_meta$css2) { + link = xml_add_child(html_head, "link") + xml_set_attrs(link, list( + rel = "stylesheet", + href = css + )) + } + + for (js in html_meta$js2) { + script = xml_add_child(html_head, "script") + xml_set_attr(script, 'src', js) + } + + body = xml_find_first(html_root, '//body') + new_body = xml_add_child(html_root, 'body') + if (!is.null(include <- html_meta$include_before)) { + include = read_html(file.path('site', include)) + include = xml_find_first(include, '//body') + for (node in xml_children(include)) + xml_add_child(new_body, node) + } + + new_body_div = xml_add_child(new_body, 'div') + xml_set_attr(new_body_div, 'class', 'body') + for (node in xml_children(body)) + xml_add_child(new_body_div, node) + + if (!is.null(include <- html_meta$include_after)) { + include = read_html(include) + include = xml_find_first(include, '//body') + for (node in xml_children(include)) + xml_add_child(new_body, node) + } + + xml_remove(body) + + for (link in xml_find_all(html_root, '//a|//link')) { + href = xml_attr(link, 'href') + if (!is.na(href)) + if (startsWith(href, '../../')) # \link[package]{topic} + xml_attr(link, 'href') <- NULL + else if (!grepl('^https?://', href)) + xml_attr(link, 'href') <- paste0('../', href) + } + + write_html(html_root, out_file) + unlink(temp_html) + + cat(sprintf(" Generated %s\n", out_file)) +} + +all_info = all_info[order(sapply(all_info, function(x) x$name), method = "radix")] + +saveRDS(all_info, "site/reference_index.rds") + +cat(sprintf("\nGenerated %d reference pages in %s/\n", length(all_info), ref_dir)) \ No newline at end of file diff --git a/site/index.Rmd b/site/index.Rmd new file mode 100644 index 0000000000..04852dda0a --- /dev/null +++ b/site/index.Rmd @@ -0,0 +1,26 @@ +--- +title: "data.table" +--- + +```{r, results='asis', echo=FALSE} +# Read README.md from repository root (one directory up from site/) +readme_file = normalizePath(file.path('..', 'README.md'), mustWork = FALSE) +if (file.exists(readme_file)) { + readme = tail(readLines(readme_file, warn = FALSE), -2) # Skip first 3 lines (title) + cat(readme, sep = '\n') +} else { + cat('README.md not found.\n') +} +``` + +## Links + +- [CRAN-like website](web/packages/data.table/index.html) +- [CRAN-like checks](web/checks/check_results_data.table.html) +- [Community blog](https://rdatatable-community.github.io/The-Raft/) + +## Citation + +```{r, echo=FALSE, results='asis'} +suppressWarnings(litedown::pkg_citation('data.table')) +``` diff --git a/site/manual.Rmd b/site/manual.Rmd new file mode 100644 index 0000000000..f1e755d603 --- /dev/null +++ b/site/manual.Rmd @@ -0,0 +1,32 @@ +--- +title: "Manual" +--- + +## All functions + +```{r, echo=FALSE, results='asis'} +# Load the reference index +ref_info = readRDS("reference_index.rds") + +format_alias = function(alias, link) { + is_simple_name = grepl("^[A-Za-z][A-Za-z0-9._]*$", alias) + alias_text = gsub("<([^>]+)>", "<\\1>", alias) + if (is_simple_name) alias_text = paste0(alias_text, "()") + sprintf("%s", link, alias_text) +} + +for (info in ref_info) { + # Create a link to the individual page + link = sprintf("reference/%s.html", info$name) + + # Format aliases as function names + if (length(info$aliases) > 0) { + formatted_aliases = vapply(info$aliases, function(alias) format_alias(alias, link), character(1)) + alias_text = paste(formatted_aliases, collapse = " ") + } else { + alias_text = format_alias(info$name, link) + } + + cat(sprintf("
\n\n %s\n\n
\n
%s
\n
\n", alias_text, info$title)) +} +``` diff --git a/site/news.Rmd b/site/news.Rmd new file mode 100644 index 0000000000..c082ec45a2 --- /dev/null +++ b/site/news.Rmd @@ -0,0 +1,11 @@ +--- +title: "News" +output: + html: + options: + toc: true +--- + +```{r, echo=FALSE, results='asis'} +litedown::pkg_news() +``` diff --git a/site/render_translations.R b/site/render_translations.R new file mode 100644 index 0000000000..f50c219a6e --- /dev/null +++ b/site/render_translations.R @@ -0,0 +1,19 @@ +files <- list.files("vignettes", "\\.Rmd$", recursive = TRUE, full.names = TRUE) +translated <- files[dirname(files) != "vignettes"] + +if (length(translated) == 0L) { + cat("No translated vignettes found\n") + quit("no") +} + +cat(sprintf("Found %d translated vignettes\n", length(translated))) + +for (f in translated) { + out <- file.path("website/articles", sub("\\.Rmd$", ".html", sub("^vignettes/", "", f))) + dir.create(dirname(out), recursive = TRUE, showWarnings = FALSE) + cat(sprintf(" Rendering: %s -> %s\n", f, out)) + src = litedown::fuse(f) + file.rename(src, out) +} + +cat("Done\n") diff --git a/vignettes/_translation_links.R b/vignettes/_translation_links.R index d384f43e39..9fbb10c287 100644 --- a/vignettes/_translation_links.R +++ b/vignettes/_translation_links.R @@ -1,6 +1,6 @@ # build a link list of alternative languages (may be character(0)) # idea is to look like 'Other languages: en | fr | de' -.write.translation.links <- function(fmt) { +.write.translation.links = function(fmt) { url = "https://rdatatable.gitlab.io/data.table/articles" path = dirname(litedown::get_context("input")) if (basename(path) == "vignettes") { diff --git a/vignettes/es/datatable-sd-usage.Rmd b/vignettes/es/datatable-sd-usage.Rmd index e5b6b15242..e15fd81ba4 100644 --- a/vignettes/es/datatable-sd-usage.Rmd +++ b/vignettes/es/datatable-sd-usage.Rmd @@ -45,11 +45,11 @@ El uso más simple de `.SD` es para la subagrupación de columnas (es decir, cua Para darle una sensación más realista, en lugar de inventar datos, carguemos algunos conjuntos de datos sobre béisbol desde la [base de datos Lahman](https://github.com/cdalzell/Lahman). En el uso típico de R, simplemente cargaríamos estos conjuntos de datos desde el paquete R `Lahman`; en este ejemplo, los hemos descargado previamente directamente desde la página de GitHub del paquete. ```{r download_lahman} -load('Teams.RData') +load('../Teams.RData') setDT(Teams) Teams -load('Pitching.RData') +load('../Pitching.RData') setDT(Pitching) Pitching ```