--- a/lib/init/variables.html +++ b/lib/init/variables.html @@ -234,12 +234,7 @@ PERMANENT
GRASS_PERL
[used during install process for generating man pages]
set Perl with path.
- -
GRASS_PROXY
-
[used during addon install/reinstall process for generating man - pages (download commit from GitHub API server and remote modules.xml file)]
- set the proxy with: GRASS_PROXY="http=<value>,ftp=<value>".
- +
GRASS_SKIP_MAPSET_OWNER_CHECK
By default it is not possible to work with MAPSETs that are not owned by current user. Setting this variable to any non-empty value --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -354,6 +354,7 @@ def download_addons_paths_file(url, response_format, *args, **kwargs): ), ) return response + except HTTPError as err: if err.code == 403 and err.msg == "rate limit exceeded": gscript.warning( @@ -2588,8 +2589,6 @@ def main(): proxy = urlrequest.ProxyHandler(PROXIES) opener = urlrequest.build_opener(proxy) urlrequest.install_opener(opener) - # Required for mkhtml.py script (get addon git commit from GitHub API server) - os.environ["GRASS_PROXY"] = options["proxy"] # define path options["prefix"] = resolve_install_prefix( --- a/utils/mkhtml.py +++ b/utils/mkhtml.py @@ -16,7 +16,6 @@ # ############################################################################# -import http import sys import os import string @@ -25,9 +24,6 @@ from datetime import datetime import locale import json import pathlib -import shutil -import subprocess -import time try: # Python 2 import @@ -35,26 +31,11 @@ try: except ImportError: # Python 3 import from html.parser import HTMLParser - -from six.moves.urllib import request as urlrequest -from six.moves.urllib.error import HTTPError, URLError - try: import urlparse except ImportError: import urllib.parse as urlparse -try: - import grass.script as gs -except ImportError: - # During compilation GRASS GIS - gs = None - -HEADERS = { - "User-Agent": "Mozilla/5.0", -} -HTTP_STATUS_CODES = list(http.HTTPStatus) - if sys.version_info[0] == 2: PY2 = True else: @@ -65,24 +46,6 @@ if not PY2: unicode = str -grass_version = os.getenv("VERSION_NUMBER", "unknown") -trunk_url = "" -addons_url = "" -if grass_version != "unknown": - major, minor, patch = grass_version.split(".") - grass_git_branch = "releasebranch_{major}_{minor}".format( - major=major, - minor=minor, - ) - base_url = "https://github.com/OSGeo" - trunk_url = "{base_url}/grass/tree/{branch}/".format( - base_url=base_url, branch=grass_git_branch - ) - addons_url = "{base_url}/grass-addons/tree/grass{major}/".format( - base_url=base_url, major=major - ) - - def _get_encoding(): encoding = locale.getdefaultlocale()[1] if not encoding: @@ -105,151 +68,6 @@ def decode(bytes_): return unicode(bytes_) -def urlopen(url, *args, **kwargs): - """Wrapper around urlopen. Same function as 'urlopen', but with the - ability to define headers. - """ - request = urlrequest.Request(url, headers=HEADERS) - return urlrequest.urlopen(request, *args, **kwargs) - - -def set_proxy(): - """Set proxy""" - proxy = os.getenv("GRASS_PROXY") - if proxy: - proxies = {} - for ptype, purl in (p.split("=") for p in proxy.split(",")): - proxies[ptype] = purl - urlrequest.install_opener( - urlrequest.build_opener(urlrequest.ProxyHandler(proxies)) - ) - - -set_proxy() - - -def download_git_commit(url, response_format, *args, **kwargs): - """Download module/addon last commit from GitHub API - - :param str url: url address - :param str response_format: content type - - :return urllib.request.urlopen or None response: response object or - None - """ - try: - response = urlopen(url, *args, **kwargs) - if not response.code == 200: - index = HTTP_STATUS_CODES.index(response.code) - desc = HTTP_STATUS_CODES[index].description - gs.fatal( - _( - "Download commit from <{url}>, return status code " - "{code}, {desc}".format( - url=url, - code=response.code, - desc=desc, - ), - ), - ) - if response_format not in response.getheader("Content-Type"): - gs.fatal( - _( - "Wrong downloaded commit file format. " - "Check url <{url}>. Allowed file format is " - "{response_format}.".format( - url=url, - response_format=response_format, - ), - ), - ) - return response - except HTTPError as err: - gs.warning( - _( - "The download of the commit from the GitHub API " - "server wasn't successful, <{}>. Commit and commit " - "date will not be included in the <{}> addon html manual " - "page.".format(err.msg, pgm) - ), - ) - except URLError: - gs.warning( - _( - "Download file from <{url}>, failed. Check internet " - "connection. Commit and commit date will not be included " - "in the <{pgm}> addon manual page.".format(url=url, pgm=pgm) - ), - ) - - -def get_last_git_commit(src_dir, is_addon, addon_path): - """Get last module/addon git commit - - :param str src_dir: module/addon source dir - :param bool is_addon: True if it is addon - :param str addon_path: addon path - - :return dict git_log: dict with key commit and date, if not - possible download commit from GitHub API server - values of keys have "unknown" string - """ - unknown = "unknown" - git_log = {"commit": unknown, "date": unknown} - datetime_format = "%A %b %d %H:%M:%S %Y" # e.g. Sun Jan 16 23:09:35 2022 - if is_addon: - grass_addons_url = ( - "https://api.github.com/repos/osgeo/grass-addons/commits?path={path}" - "&page=1&per_page=1&sha=grass{major}".format( - path=addon_path, - major=major, - ) - ) # sha=git_branch_name - else: - core_module_path = os.path.join( - *(set(src_dir.split(os.path.sep)) ^ set(topdir.split(os.path.sep))) - ) - grass_modules_url = ( - "https://api.github.com/repos/osgeo/grass/commits?path={path}" - "&page=1&per_page=1&sha={branch}".format( - branch=grass_git_branch, - path=core_module_path, - ) - ) # sha=git_branch_name - - if shutil.which("git"): - if os.path.exists(src_dir): - git_log["date"] = time.ctime(os.path.getmtime(src_dir)) - stdout, stderr = subprocess.Popen( - args=["git", "log", "-1", src_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ).communicate() - stdout = decode(stdout) - stderr = decode(stderr) - - if stderr and "fatal: not a git repository" in stderr: - response = download_git_commit( - url=grass_addons_url if is_addon else grass_modules_url, - response_format="application/json", - ) - if response: - commit = json.loads(response.read()) - if commit: - git_log["commit"] = commit[0]["sha"] - git_log["date"] = datetime.strptime( - commit[0]["commit"]["author"]["date"], - "%Y-%m-%dT%H:%M:%SZ", - ).strftime(datetime_format) - else: - if stdout: - commit = stdout.splitlines() - git_log["commit"] = commit[0].split(" ")[-1] - commit_date = commit[2].lstrip("Date:").strip() - git_log["date"] = commit_date.rsplit(" ", 1)[0] - return git_log - - html_page_footer_pages_path = ( os.getenv("HTML_PAGE_FOOTER_PAGES_PATH") if os.getenv("HTML_PAGE_FOOTER_PAGES_PATH") @@ -261,6 +79,14 @@ pgm = sys.argv[1] src_file = "%s.html" % pgm tmp_file = "%s.tmp.html" % pgm +grass_version = os.getenv("VERSION_NUMBER", "unknown") +trunk_url = "" +addons_url = "" +if grass_version != "unknown": + major, minor, patch = grass_version.split(".") + trunk_url = f"https://github.com/OSGeo/grass/tree/releasebranch_{major}_{minor}/" + addons_url = f"https://github.com/OSGeo/grass-addons/tree/grass{major}/" + header_base = """ @@ -295,9 +121,6 @@ sourcecode = string.Template( ${PGM} source code (history)

-

- ${DATE_TAG} -

""" ) @@ -624,7 +447,6 @@ else: source_url = addons_url pgmdir = os.path.sep.join(curdir.split(os.path.sep)[-3:]) url_source = "" -addon_path = None if os.getenv("SOURCE_URL", ""): addon_path = get_addon_path() if addon_path: @@ -655,23 +477,11 @@ if index_name: else: url_log = url_source.replace(tree, commits) - git_commit = get_last_git_commit( - src_dir=curdir, - addon_path=addon_path if addon_path else None, - is_addon=True if addon_path else False, - ) - if git_commit["commit"] == "unknown": - date_tag = "Accessed: {date}".format(date=git_commit["date"]) - else: - date_tag = "Latest change: {date} in commit: {commit}".format( - date=git_commit["date"], commit=git_commit["commit"] - ) sys.stdout.write( sourcecode.substitute( URL_SOURCE=url_source, PGM=pgm, URL_LOG=url_log, - DATE_TAG=date_tag, ) ) sys.stdout.write(