From 37b51b2a5fd3644c6b1842cbe504da6e9bbc588a Mon Sep 17 00:00:00 2001 From: "Lucas C. Villa Real" Date: Sat, 30 Jun 2018 01:02:39 -0300 Subject: Improved dependency search on rpmfind.net: we now include the distro code (i.e., fc26, fc27, etc) and have a placeholder to check versions when a range filter is given. We can probably use some of the code from Scripts' FindDependencies.c here. --- Functions/DEB | 6 +++- Functions/RPM | 15 ++++++++-- bin/RPMFinder | 76 +++++++++++++++++++++++++++++++++++++++---------- bin/ThirdPartyInstaller | 34 +++++++++++++++------- 4 files changed, 101 insertions(+), 30 deletions(-) diff --git a/Functions/DEB b/Functions/DEB index 1c0e498..5d59135 100644 --- a/Functions/DEB +++ b/Functions/DEB @@ -17,7 +17,7 @@ function thirdparty_arch() { fi } -function thirdparty_distribution() { +function thirdparty_distribution_name() { local debfile="$1" local distro=$(dpkg-deb --field "$debfile" "Distribution") if [ -z "$distro" ] @@ -26,6 +26,10 @@ function thirdparty_distribution() { fi } +function thirdparty_distribution_code() { + return +} + function thirdparty_dependencies() { local debfile="$1" local deps=$(dpkg-deb --field "$debfile" "Depends") diff --git a/Functions/RPM b/Functions/RPM index 635f643..a166fb0 100644 --- a/Functions/RPM +++ b/Functions/RPM @@ -11,11 +11,16 @@ function thirdparty_arch() { rpminfo --arch "$rpmfile" } -function thirdparty_distribution() { +function thirdparty_distribution_name() { local rpmfile="$1" rpminfo --distribution "$rpmfile" } +function thirdparty_distribution_code() { + local rpmfile="$1" + echo "$rpmfile" | awk -F. '{print $(NF-2)}' +} + function thirdparty_dependencies() { local rpmfile="$1" rpminfo --dependencies "$rpmfile" @@ -64,8 +69,12 @@ function thirdparty_url() { function thirdparty_search_remotedb() { local path="$1" local arch="$2" - local distro="$3" - RPMFinder --path="$path" --arch="$arch" --distro="$distro" + local distroname="$3" + local distrocode="$4" + local result=$(RPMFinder --path="$path" --arch="$arch" --distroname="$distroname" --distrocode="$distrocode") + [ -z "$result" ] && [ "$arch" = "noarch" ] && result=$(RPMFinder --path="$path" --arch="$(uname -m)" --distroname="$distroname" --distrocode="$distrocode") + [ -z "$result" ] && return + echo "$result" } function thirdparty_uncompress() { diff --git a/bin/RPMFinder b/bin/RPMFinder index 995bb8a..1b8bf2f 100755 --- a/bin/RPMFinder +++ b/bin/RPMFinder @@ -9,6 +9,7 @@ # Released under the GNU GPL version 2 or above. import os +import re import sys import argparse import subprocess @@ -34,53 +35,98 @@ class RPMFind_Parser(HTMLParser): if len(self.tags) and self.tags[-1] == "a" and data.find(".rpm") >= 0: self.candidates.append(data) - def get_pkgname(self): + def get_pkgnames(self): if len(self.candidates) == 0: return "" - name = os.path.commonprefix(self.candidates) - if name.endswith("-"): - name = name[:-1] - return name + return self.candidates class RPMFinder: - def find(self, path, arch, distro): + def find(self, path, arch, distroname, distrocode): ''' Searches rpmfind.net for a given file. Arch and distro can - be provided to reduce the search scope. Returns the package + be provided to narrow the search scope. Returns the package name on success or an empty string if no matches were found. ''' self.path = path self.arch = arch - self.distro = distro - return self.__search_rpmfind_net() + self.distroname = distroname.replace(" ", "+") + self.distrocode = distrocode + self.name = path + for token in [">=", ">", "<=", "<", "="]: + self.name = self.name.replace(token, " ") + self.name = self.name.split(" ")[0] + + matches = self.__search_rpmfind_net() + if len(matches) == 0: + return [] + if any(op in self.path for op in [">", "<", "="]): + op, version = self.__get_op_and_version() + return self.__filter(matches, op, version) + else: + return matches + + def __filter(self, matches, op, version): + filtered = [] + for match in matches: + pkg_version = match.replace(self.name, "").replace("{0}.{1}.rpm".format(self.distrocode, self.arch), "").strip("-").strip(".") + sys.stderr.write("package {0}: {1} {2} {3}?\n".format(match, pkg_version, op, version)) + + # TODO: filter based on op and version + if len(filtered) == 0: + filtered.append(match) + return filtered + + def __get_op_and_version(self): + if ">=" in self.path: + return ">=", self.path.split(">=")[1].strip() + elif ">" in self.path: + return ">", self.path.split(">")[1].strip() + elif "<=" in self.path: + return "<=", self.path.split("<=")[1].strip() + elif "<" in self.path: + return "<", self.path.split("<")[1].strip() + elif "=" in self.path: + return "=", self.path.split("=")[1].strip() + else: + sys.stderr.write("could not extract op and version from {}\n".format(self.path)) + return None, None def __search_rpmfind_net(self): path = self.path.replace("/", "%2F") - arch = self.arch baseuri = "http://rpmfind.net/linux/rpm2html/search.php" - query = "?query={0}&submit=Search+...&system=&arch={1}".format(path, arch) + query = "?query={0}&submit=Search+...&system={1}&arch={2}".format(path, self.distroname, self.arch) html = subprocess.check_output(["wget", "--quiet", "{0}{1}".format(baseuri, query), "-O", "-"]) + # Compile a regex that catches package names derived from the basename given by self.path. + # Example: perl-DBICx when perl-DBI is wanted. + regex = re.compile(r"{0}\-[0-9]+.*{1}.{2}.rpm".format(self.name, self.distrocode, self.arch)) + htmlparser = RPMFind_Parser() htmlparser.feed(str(html)) - return htmlparser.get_pkgname() + pkgnames = htmlparser.get_pkgnames() + result = [] + for pkgname in pkgnames: + if regex.match(pkgname): + result.append(pkgname) + return result def main(): argparser = argparse.ArgumentParser(argument_default="") argparser.add_argument("--path", type=str, help="File name to search for in the remote RPM databases") argparser.add_argument("--arch", type=str, help="Architecture (optional)") - argparser.add_argument("--distro", type=str, help="Distribution (optional)") + argparser.add_argument("--distroname", type=str, help="Distribution name (optional)") + argparser.add_argument("--distrocode", type=str, help="Distribution code (optional)") args = argparser.parse_args() if len(args.path) == 0: argparser.print_help() sys.exit(1) - pkgname = RPMFinder().find(args.path, args.arch, args.distro) + pkgname = RPMFinder().find(args.path, args.arch, args.distroname, args.distrocode) if len(pkgname): - print(pkgname) + print(pkgname[0]) if __name__ == "__main__": main() diff --git a/bin/ThirdPartyInstaller b/bin/ThirdPartyInstaller index caaf8b9..76d1755 100755 --- a/bin/ThirdPartyInstaller +++ b/bin/ThirdPartyInstaller @@ -9,7 +9,7 @@ Import OptionParser ### Options ################################################################### scriptDescription="Install RPM and DEB packages on GoboLinux." -scriptCredits="Copyright (C) Lucas C. Villa Real, 2016,2017 - Released under the GNU GPL." +scriptCredits="Copyright (C) Lucas C. Villa Real, 2016-2018 - Released under the GNU GPL." helpOnNoArguments=yes scriptUsage=" [file.rpm]" scriptExample="xispita-2.0.3-1.x86_64.rpm" @@ -223,7 +223,7 @@ function populate_dependencies() { then Log_Verbose "Skipping dependency passed as input file: $dependency" else - depinfo=$(lookup_pkgname "$dependency") + depinfo=$(lookup_pkgname "$inputfile" "$dependency") if [ "$depinfo" ] then echo "$depinfo" else echo "# Unresolved dependency: $dependency" @@ -237,7 +237,7 @@ function populate_resources() { local arch=$(thirdparty_arch "$inputfile") local description=$(thirdparty_description "$inputfile") local release=$(thirdparty_release "$inputfile") - local distro=$(thirdparty_distribution "$inputfile") + local distro=$(thirdparty_distribution_name "$inputfile") Log_Normal "Populating Resources." mkdir -p Resources @@ -307,7 +307,8 @@ function take_dependency_from_path() { local symbol="$3" local fullpath="$(readlink -f $path)" local arch=$(thirdparty_arch "$inputfile") - local distro=$(thirdparty_distribution "$inputfile") + local distro=$(thirdparty_distribution_name "$inputfile") + local distrocode=$(thirdparty_distribution_code "$inputfile") local depname= local depversion= @@ -364,7 +365,7 @@ function take_dependency_from_path() { if Boolean "web" then Log_Normal "Searching the remote $(thirdparty_backend) database for the package hosting $originalpath" - depname=$(thirdparty_search_remotedb "$originalpath" "$arch" "$distro") + depname=$(thirdparty_search_remotedb "$originalpath" "$arch" "$distro" "$distrocode") if [ "$depname" ] then # TODO: we could now lookup the GoboLinux recipe store to find whether we @@ -377,7 +378,8 @@ function take_dependency_from_path() { } function lookup_pkgname() { - local dependency="$1" + local inputfile="$1" + local dependency="$2" local pkgname=$(echo "$dependency" | cut -d'(' -f1) # GoboLinux doesn't have "devel" packages like most mainstream distros do @@ -393,20 +395,30 @@ function lookup_pkgname() { fi done - # Query the GoboLinux recipe store. if Boolean "web" then + # Query the GoboLinux recipe store. local recipeurl=$(FindPackage -t recipe "${pkgname}" || FindPackage -t recipe "${goboname}") if [ "$recipeurl" ] then # TODO we're potentially discarding the wanted version(s) of the given dep echo "$(basename $recipeurl | sed 's,\(.*\)--.*--.*,\1,g')" && return 0 fi - fi - - # TODO Query the remote RPM database for $pkgname - # TODO Query the remote DEB database for $pkgname + # Query the remote RPM database for $pkgname + local arch=$(thirdparty_arch "$inputfile") + local distro=$(thirdparty_distribution_name "$inputfile") + local distrocode=$(thirdparty_distribution_code "$inputfile") + Log_Normal "Searching the remote $(thirdparty_backend) database for the package hosting $pkgname" + local depname=$(thirdparty_search_remotedb "$pkgname" "$arch" "$distro" "$distrocode") + if [ "$depname" ] + then + # TODO: download and install the RPM + echo "$depname" && return 0 + fi + + # TODO Query the remote DEB database for $pkgname + fi return 1 } -- cgit v1.1