[core] prebake: allow packages to be fetched from web instead of local dirs only

From: Jacob Kjaergaard <jacob.kjaergaard@prevas.dk>

set PREBAKE_URL to point to your prebaked packages url prefix e.g:
PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked
packages are located

http://myserver.com/packages/

the directory structure under "packages" should ressemble the structure
under tmp/packages/
---
 conf/oe-lite.conf   |    1 +
 conf/oelayout.conf  |    3 +++
 doc/manual.txt      |    2 ++
 doc/prebake.txt     |   22 ++++++++++++++++++++++
 lib/oelite/baker.py |   27 +++++++++++++++++++++++++++
 5 files changed, 55 insertions(+)
 create mode 100644 doc/prebake.txt

<jabk@prevas.dk> writes:

> From: Jacob Kjaergaard <jacob.kjaergaard@prevas.dk>
>
> set PREBAKE_URL to point to your prebaked packages url prefix e.g:
> PREBAKE_URL_CACHE_DIR can be set to pin point where the fetched prebaked

I guess that should be PREBAKE_CACHE_DIR instead.

> packages are located
>
> http://myserver.com/packages/
>
> the directory structure under "packages" should ressemble the structure
> under tmp/packages/
> ---
>  conf/oe-lite.conf   |    1 +
>  conf/oelayout.conf  |    3 +++
>  doc/manual.txt      |    2 ++
>  doc/prebake.txt     |   22 ++++++++++++++++++++++
>  lib/oelite/baker.py |   27 +++++++++++++++++++++++++++
>  5 files changed, 55 insertions(+)
>  create mode 100644 doc/prebake.txt
>
> diff --git a/conf/oe-lite.conf b/conf/oe-lite.conf
> index b1e23b2..4468b37 100644
> --- a/conf/oe-lite.conf
> +++ b/conf/oe-lite.conf
> @@ -41,6 +41,7 @@ DEFAULT_RELAX[nohash] = "1"
>  PREBAKE_PATH[nohash] = "1"
>  
>  PARALLEL_MAKE[nohash] = True
> +PREBAKE_URL[nohash] = True
>  export PATH
>  
>  export LD_LIBRARY_PATH
> diff --git a/conf/oelayout.conf b/conf/oelayout.conf
> index 535cb6f..405e4a2 100644
> --- a/conf/oelayout.conf
> +++ b/conf/oelayout.conf
> @@ -57,6 +57,9 @@ IMAGE_DEPLOY_DIR[nohash] = True
>  PACKAGE_DEPLOY_DIR 	 = "${TMPDIR}/packages"
>  PACKAGE_DEPLOY_DIR[nohash] = True
>  
> +PREBAKE_CACHE_DIR 	      = "${TMPDIR}/prebake"
> +PREBAKE_CACHE_DIR[nohash] = True
> +
>  # Recipe directory layout
>  FILESPATHBASE	= "${FILE_DIRNAME}"
>  FILESPATHBASE[emit] = ""
> diff --git a/doc/manual.txt b/doc/manual.txt
> index 2bcd230..e0fe093 100644
> --- a/doc/manual.txt
> +++ b/doc/manual.txt
> @@ -40,6 +40,8 @@ include::quickstart.txt[]
>  // Overview
>  //include::overview.txt[]
>  
> +include::prebake.txt[]
> +
>  // Build Configuration
>  //include::buildcfg.txt[]
>  
> diff --git a/doc/prebake.txt b/doc/prebake.txt
> new file mode 100644
> index 0000000..da1a1f6
> --- /dev/null
> +++ b/doc/prebake.txt

Please word wrap the long lines in documentation files to make them more
readable in text editors.

> @@ -0,0 +1,22 @@
> +Using prebaked packages
> +=======================
> +
> +This section describes how to use packages build on another machine
> +available via http.
> +
> +
> +First you need to setup a http server that serves the prebaked
> packages from another machine. The baked prebakes will be in
> `${TMPDIR}/packages` so setup the  server to make files available
> online as you prefer.
> +
> +Secondly, you need to set PREBAKE_URL to point to your prebaked packages url prefix e.g:
> +PREBAKE_CACHE_DIR can be set to pin point where the fetched prebaked packages are located
> +
> +----
> +#cat conf/site.conf
> +PREBAKE_URL="http://myserver.com/packages/"
> +PREBAKE_CACHE_DIR="<path-to-temp-storage>"
> +----
> +
> +Note, that the directory structure under "packages" should ressemble
> the structure

resemble or match?

> +under `${TMPDIR}/packages`.
> +
> +Presently, 32-bit machines should bake prebaked packages that should be used on 32-bit machines as well (and likewise for 64-bit).

Perhaps some words of warnings that prebakes might, or might not, work
on different host OS versions would be appropriate here.

> diff --git a/lib/oelite/baker.py b/lib/oelite/baker.py
> index 20cf146..476b22a 100644
> --- a/lib/oelite/baker.py
> +++ b/lib/oelite/baker.py
> @@ -12,6 +12,10 @@ import oelite.item
>  from oelite.parse import *
>  from oelite.cookbook import CookBook
>  
> +import oelite.fetch
> +
> +import bb.utils
> +
>  import sys
>  import os
>  import glob
> @@ -437,6 +441,9 @@ class OEliteBaker:
>          # check for availability of prebaked packages, and set package
>          # filename for all packages.
>          depend_packages = self.runq.get_depend_packages()
> +        url_prefix = self.config.get("PREBAKE_URL")
> +        if url_prefix is not None:
> +            info("Trying to use prebakes from url: %s"%(url_prefix))
>          for package in depend_packages:
>              # FIXME: skip this package if it is to be rebuild
>              prebake = self.find_prebaked_package(package)
> @@ -608,6 +615,7 @@ class OEliteBaker:
>      def find_prebaked_package(self, package):
>          """return full-path filename string or None"""
>          package_deploy_dir = self.config.get("PACKAGE_DEPLOY_DIR")
> +        prebake_url_cache_dir = self.config.get("PREBAKE_CACHE_DIR")
>          if not package_deploy_dir:
>              die("PACKAGE_DEPLOY_DIR not defined")
>          if self.options.prebake:
> @@ -615,6 +623,7 @@ class OEliteBaker:
>              if prebake_path:
>                  prebake_path = prebake_path.split(":")
>              prebake_path.insert(0, package_deploy_dir)
> +            prebake_path.insert(0, prebake_url_cache_dir)
>          else:
>              prebake_path = [package_deploy_dir]
>          debug("package=%s"%(repr(package)))
> @@ -628,6 +637,7 @@ class OEliteBaker:
>              raise NoSuchPackage()
>          filename = "%s_%s_%s.tar"%(package.name, recipe.version, metahash)
>          debug("prebake_path=%s"%(prebake_path))
> +        #test local paths first
>          for base_dir in prebake_path:
>              path = os.path.join(
>                  base_dir,
> @@ -638,6 +648,23 @@ class OEliteBaker:
>              if os.path.exists(path):
>                  debug("found prebake: %s"%(path))
>                  return path
> +        #then test URLs from PREBAKE_URL
> +        url_prefix = self.config.get("PREBAKE_URL")
> +        if url_prefix is not None:
> +            package_path =os.path.join(
> +                    package.type,
> +                    package.arch + (package.recipe.meta.get("EXTRA_ARCH") or ""),
> +                    filename)
> +            downloaded_file =os.path.join(
> +                    prebake_url_cache_dir,
> +                    package_path)
> +            url = os.path.join(url_prefix, package_path)
> +            if oelite.fetch.url.grab(url, downloaded_file, timeout=1, retry=1):
> +                if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:
> +                    debug("using prebake from web: %s"%(url))
> +                    return downloaded_file
> +                else:
> +                    os.unlink(downloaded_file)
>          return None
>  
>

/Esben