mirror of
https://github.com/saymrwulf/uhd.git
synced 2026-05-14 20:58:09 +00:00
images: Update scripts for new release model
- populate_images.py: Made Py3k safe - uhdimgs.py: Make Py3k-safe - create_imgs_package.py: Rewrote to build release images packages - make_zip.sh: Skip CMake, just zip up the files - Updated README to reflect all the changes
This commit is contained in:
Martin Braun
parent
9591c93dd5
commit
174bde6cc1
6 changed files with 96 additions and 261 deletions
|
|
@ -1,52 +0,0 @@
|
|||
#
|
||||
# Copyright 2010-2012 Ettus Research LLC
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
|
||||
########################################################################
|
||||
# Setup Project
|
||||
########################################################################
|
||||
#OPTION(UHD_PATCH_OVERRIDE "Use this string to force a patch level version string." OFF)
|
||||
MESSAGE(STATUS ${UHD_PATCH_OVERRIDE})
|
||||
IF(DEFINED UHD_PATCH_OVERRIDE)
|
||||
SET(UHD_VERSION_PATCH_OVERRIDE ${UHD_PATCH_OVERRIDE})
|
||||
ENDIF(DEFINED UHD_PATCH_OVERRIDE)
|
||||
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
|
||||
PROJECT(UHD-images NONE)
|
||||
LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../host/cmake/Modules)
|
||||
INCLUDE(UHDVersion) #sets version variables (used below)
|
||||
EXECUTE_PROCESS(COMMAND "date" OUTPUT_VARIABLE DATETIME_NOW OUTPUT_STRIP_TRAILING_WHITESPACE) #sets the date
|
||||
|
||||
########################################################################
|
||||
# Setup CPack
|
||||
########################################################################
|
||||
SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Ettus Research - USRP Hardware Driver Images")
|
||||
SET(CPACK_PACKAGE_VENDOR "Ettus Research LLC")
|
||||
SET(CPACK_PACKAGE_CONTACT "support@ettus.com")
|
||||
SET(CPACK_PACKAGE_VERSION ${UHD_VERSION})
|
||||
SET(CPACK_PACKAGE_FILE_NAME "uhd-images_${UHD_VERSION}")
|
||||
SET(CPACK_DEBIAN_PACKAGE_ARCHITECTURE "all")
|
||||
INCLUDE(CPack) #include after setting vars
|
||||
MESSAGE(STATUS "Version: ${UHD_VERSION}")
|
||||
|
||||
########################################################################
|
||||
# Install Images
|
||||
########################################################################
|
||||
|
||||
#tag the images with a version number (something identifiable)
|
||||
FILE(WRITE ${CMAKE_SOURCE_DIR}/images/${UHD_VERSION_MAJOR}.${UHD_VERSION_API}.${UHD_VERSION_ABI}.${UHD_VERSION_PATCH}.tag "${UHD_VERSION}\n${DATETIME_NOW}\n")
|
||||
INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/images DESTINATION share/uhd)
|
||||
INSTALL(FILES ${CMAKE_SOURCE_DIR}/../host/LICENSE DESTINATION share/uhd/images)
|
||||
INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/manifest.txt DESTINATION share/uhd/images)
|
||||
|
|
@ -1,83 +1,20 @@
|
|||
UHD Firmware and FPGA Images Builder
|
||||
===============================================================
|
||||
|
||||
The images directory contains the following:
|
||||
* A Makefile for building firmware and FPGA images
|
||||
* Scripts to load all the binaries for this current commit, as
|
||||
well as create a new images package from the images in subdir
|
||||
images/
|
||||
The images directory is an aid to prepare images packages.
|
||||
|
||||
This provides a clean and organized way to build all of the firmware and FPGA
|
||||
images, the source code for which is in the `firmware` and `fpga` directories
|
||||
one level above this, and also maintains a linkage between images and git commits.
|
||||
Building the actual FPGA images is not handled with these tools.
|
||||
|
||||
Building the binaries
|
||||
---------------------
|
||||
## Making image packages
|
||||
|
||||
The Makefile and build systems for the images are *probably* Unix-specific.
|
||||
It's best to build the images on a Unix system with standard build tools. The
|
||||
CMake package target will create an installable images package for your system.
|
||||
At any point in time, when the FPGA images change, you will need to create new
|
||||
FPGA images packages. Use the `package_images.py` script for that purpose.
|
||||
|
||||
__To build the images (unix):__
|
||||
## Create image tarballs for releases
|
||||
|
||||
1. `make clean`
|
||||
2. `make images`
|
||||
If you're on a release tag and want to create images packages for uploading to
|
||||
github, run
|
||||
|
||||
__Fedora note:__
|
||||
|
||||
The sdcc binaries are prefixed with "sdcc-" which breaks the build.
|
||||
However, /usr/libexec/sdcc contains properly named sdcc binaries.
|
||||
`export PATH=${PATH}:/usr/libexec/sdcc`
|
||||
|
||||
|
||||
Updating binaries
|
||||
-----------------
|
||||
|
||||
This goes two ways:
|
||||
|
||||
1. Loading the correct binaries for this commit
|
||||
2. Updating the binary package on this branch
|
||||
|
||||
|
||||
### Loading the correct binaries ###
|
||||
|
||||
If you check out a branch or commit, you might want to use the exact same
|
||||
binaries that were used when this branch or commit was generated.
|
||||
To do this, run `populate_images.py`. This will either download the correct
|
||||
images package from a web server or from a local directory if
|
||||
`UHD_IMAGES_BASE_URL` is set.
|
||||
|
||||
### Updating the binaries ###
|
||||
|
||||
If you have commited changes to this branch that require new images, you
|
||||
should probably update those.
|
||||
Simply copy the new image binaries into images/. If necessary, run
|
||||
`populate_images.py` before you do any of this, so all the untouched
|
||||
images are the correct version. Then, run `create_imgs_package.py --commit "COMMIT MSG"`
|
||||
to create a new ZIP file and commit the info.
|
||||
If `UHD_IMAGES_BASE_URL` is set and is a local directory, it will move
|
||||
the ZIP file to this directory after creating it.
|
||||
|
||||
### Updating the binaries ###
|
||||
|
||||
Typical workflow:
|
||||
|
||||
1. Check out a branch or commit (git checkout FOO)
|
||||
2. Update the images/ subdir (`populate_images.py`)
|
||||
3. Do some coding, and commit those changes (`git commit`)
|
||||
4. Copy new binaries to images/
|
||||
5. Commit the new binaries: `create_imgs_package.py --commit "Updated images on branch X"`
|
||||
|
||||
### The CPack system ###
|
||||
|
||||
Underlying `create_imgs_package.py` is a CPack system, which can be manually
|
||||
invoked by:
|
||||
|
||||
1. `mkdir build`
|
||||
2. `cd build`
|
||||
3. `cmake -DCPACK_GENERATOR=<type> ../`
|
||||
4. `make package`
|
||||
|
||||
The package generator types are described here:
|
||||
http://www.cmake.org/Wiki/CMake:CPackPackageGenerators
|
||||
$ ./create_imgs_package.py
|
||||
|
||||
It'll take a few minutes to download and zip them all up.
|
||||
|
|
|
|||
|
|
@ -19,86 +19,66 @@
|
|||
Command-line utility to create a .zip-file with the current image set.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
import re
|
||||
import os
|
||||
import uhdimgs
|
||||
import glob
|
||||
import subprocess
|
||||
import argparse
|
||||
import shutil
|
||||
|
||||
def clear_img_dir(img_root_dir):
|
||||
""" Removes non-image files from the images dir """
|
||||
globs = ["*.tag", "LICENSE"]
|
||||
for the_glob in globs:
|
||||
for filename in glob.iglob(os.path.join(img_root_dir, the_glob)):
|
||||
print 'Removing file from images directory: ', filename
|
||||
os.unlink(filename)
|
||||
|
||||
def get_zipfilename_from_cpack_output(cpoutput):
|
||||
""" Parses the output of the ZIP-file creating script
|
||||
and scrapes the actual file name. """
|
||||
regex = re.compile("\/build\/(?P<filename>[^\/]+\.zip)")
|
||||
results = regex.search(cpoutput)
|
||||
return results.group('filename')
|
||||
import uhdimgs
|
||||
|
||||
def parse_args():
|
||||
""" Parse args, duh """
|
||||
parser = argparse.ArgumentParser(description='Link the current set of images to this commit.')
|
||||
parser.add_argument('--commit', default=None,
|
||||
help='Supply a commit message to the changes to host/CMakeLists.txt.')
|
||||
parser.add_argument('-r', '--release-mode', default="",
|
||||
help='Specify UHD_RELEASE_MODE. Typically "release" or "rc1" or similar.')
|
||||
parser.add_argument('--skip-edit', default=False, action='store_true',
|
||||
help='Do not edit the CMakeLists.txt file.')
|
||||
parser.add_argument('--skip-move', default=False, action='store_true',
|
||||
help='Do not move the archives after creating them.')
|
||||
parser.add_argument('--patch', help='Override patch version number.')
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--version',
|
||||
help="Specify version. Will detect from tag otherwise."
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
def move_zip_to_repo(base_url, zipfilename):
|
||||
final_destination = os.path.join(base_url, zipfilename)
|
||||
if os.path.exists(final_destination):
|
||||
print "WARNING: A file with name {0} is already in the images repository.".format(zipfilename)
|
||||
print "Overwrite? [y/N]",
|
||||
ans = raw_input()
|
||||
if ans.strip().upper() != 'Y':
|
||||
return
|
||||
os.unlink(final_destination)
|
||||
shutil.move(zipfilename, base_url)
|
||||
def download_images(img_root_dir):
|
||||
"""
|
||||
Run the images downloader
|
||||
"""
|
||||
import populate_images
|
||||
populate_images.download_images(img_root_dir)
|
||||
|
||||
def get_version():
|
||||
"""
|
||||
Figure out version based on tag.
|
||||
"""
|
||||
try:
|
||||
git_cmd = ['git', 'describe', '--abbrev=0', '--tags']
|
||||
git_output = subprocess.check_output(git_cmd)
|
||||
except subprocess.CalledProcessError as ex:
|
||||
print(ex.output)
|
||||
exit(1)
|
||||
print("Detected tag: {}".format(git_output))
|
||||
version_mobj = re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]', git_output)
|
||||
if version_mobj is None:
|
||||
print("Error: Failure to resolve version from tag!")
|
||||
exit(1)
|
||||
return version_mobj.group(0)
|
||||
|
||||
def main():
|
||||
" Go, go, go! "
|
||||
""" Go, go, go! """
|
||||
args = parse_args()
|
||||
img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images')
|
||||
os.chdir(uhdimgs.get_images_dir())
|
||||
print "== Clearing out the images directory..."
|
||||
clear_img_dir(img_root_dir)
|
||||
print "== Creating archives..."
|
||||
cpack_cmd = ["./make_zip.sh",]
|
||||
cpack_cmd.append(args.release_mode)
|
||||
if args.patch is not None:
|
||||
cpack_cmd.append("-DUHD_PATCH_OVERRIDE={}".format(args.patch))
|
||||
print("== Clearing out the images directory...")
|
||||
shutil.rmtree(img_root_dir)
|
||||
print("== Downloading images...")
|
||||
download_images(img_root_dir)
|
||||
print("== Determining version...")
|
||||
version = args.version if args.version is not None else get_version()
|
||||
print("Version string: {}".format(version))
|
||||
print("== Creating archives...")
|
||||
archive_cmd = ["./make_zip.sh", version]
|
||||
try:
|
||||
cpack_output = subprocess.check_output(cpack_cmd)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print e.output
|
||||
raise SystemExit, 1
|
||||
zipfilename = get_zipfilename_from_cpack_output(cpack_output)
|
||||
print "Filename: ", zipfilename
|
||||
print "== Calculating MD5 sum of ZIP archive..."
|
||||
md5 = uhdimgs.md5_checksum(zipfilename)
|
||||
print 'MD5: ', md5
|
||||
base_url = uhdimgs.get_base_url()
|
||||
if not args.skip_move and uhdimgs.base_url_is_local(base_url) and os.access(base_url, os.W_OK):
|
||||
print "== Moving ZIP file to {0}...".format(base_url)
|
||||
move_zip_to_repo(base_url, zipfilename)
|
||||
print "== Updating CMakeLists.txt..."
|
||||
uhdimgs.update_main_cmake_file(md5, zipfilename)
|
||||
if args.commit is not None:
|
||||
print "== Committing changes..."
|
||||
subprocess.check_call(['git', 'commit', '-m', args.commit, uhdimgs.get_cmake_main_file()])
|
||||
print "== Done!"
|
||||
subprocess.call(archive_cmd)
|
||||
except subprocess.CalledProcessError as ex:
|
||||
print(ex.output)
|
||||
exit(1)
|
||||
print("== Done!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -11,46 +11,31 @@ if [ ! -e 'images' ]; then
|
|||
fi
|
||||
|
||||
# Remove cruft before proceeding:
|
||||
if [ -e 'build' ]; then
|
||||
echo 'Please remove build subdirectory before proceeding.'
|
||||
exit 1
|
||||
fi
|
||||
if [ -e "images/LICENSE" ]; then
|
||||
rm images/LICENSE
|
||||
fi
|
||||
TAGFILES=`ls images/*.tag 2>/dev/null`
|
||||
if [ -n "$TAGFILES" ]; then
|
||||
rm $TAGFILES
|
||||
fi
|
||||
|
||||
# Enter build dir
|
||||
mkdir build
|
||||
cd build
|
||||
# Copy LICENSE file
|
||||
cp ../host/LICENSE images/
|
||||
|
||||
# Run the CPack process (ZIP file)
|
||||
cmake .. -DCPACK_GENERATOR=ZIP -DUHD_RELEASE_MODE="$1" $2 ..
|
||||
make package
|
||||
mv uhd-images*.zip ..
|
||||
ARCHIVE_SUFFIX=$1
|
||||
ARCHIVE_NAME="uhd-images_${ARCHIVE_SUFFIX}"
|
||||
echo "Creating images archive: ${ARCHIVE_NAME}"
|
||||
|
||||
# Run the CPack process (tarball)
|
||||
cmake .. -DCPACK_GENERATOR=TGZ -DUHD_RELEASE_MODE="$1" $2 ..
|
||||
make package
|
||||
mv uhd-images*.tar.gz ..
|
||||
cp -r images $ARCHIVE_NAME
|
||||
|
||||
# Move images to here and clean up after us:
|
||||
cd ..
|
||||
rm -r build
|
||||
rm images/*.tag
|
||||
# Now zip 'em up:
|
||||
echo "Creating ZIP archive..."
|
||||
zip -r $ARCHIVE_NAME.zip $ARCHIVE_NAME
|
||||
echo "Creating tar.gz archive..."
|
||||
tar zcvf $ARCHIVE_NAME.tar.gz $ARCHIVE_NAME
|
||||
echo "Creating tar.xz archive..."
|
||||
tar Jcvf $ARCHIVE_NAME.tar.xz $ARCHIVE_NAME
|
||||
ALL_ARCHIVES=`ls $ARCHIVE_NAME.*`
|
||||
sha256sum $ALL_ARCHIVES > $ARCHIVE_NAME.sha256
|
||||
md5sum $ALL_ARCHIVES > $ARCHIVE_NAME.md5
|
||||
|
||||
TGZ_ARCHIVE_NAME=`ls *.tar.gz | tail -n1`
|
||||
# Clean up archive directory
|
||||
rm -r $ARCHIVE_NAME
|
||||
|
||||
# CMake can't do xz, so do it by hand if possible
|
||||
XZ_EXECUTABLE=`which xz`
|
||||
if [ $? -eq 0 ]; then
|
||||
XZ_ARCHIVE_NAME=`echo $TGZ_ARCHIVE_NAME | sed "s/gz\>/xz/"`
|
||||
echo "Writing .xz tarball to $XZ_ARCHIVE_NAME ..."
|
||||
gunzip --to-stdout $TGZ_ARCHIVE_NAME | xz - > $XZ_ARCHIVE_NAME
|
||||
fi
|
||||
|
||||
MD5_FILE_NAME=`echo $TGZ_ARCHIVE_NAME | sed "s/tar.gz\>/md5/"`
|
||||
md5sum uhd-images* > $MD5_FILE_NAME
|
||||
|
|
|
|||
|
|
@ -1,47 +1,41 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2014 Ettus Research LLC
|
||||
# Copyright 2018 Ettus Research, a National Instruments Company
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
"""
|
||||
Populates the current directory with a valid set of binaries for the
|
||||
current commit.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import subprocess
|
||||
import uhdimgs
|
||||
|
||||
def main():
|
||||
def download_images(img_root_dir=None):
|
||||
" Go, go, go! "
|
||||
# Switch to correct dir
|
||||
img_root_dir = os.path.join(uhdimgs.get_images_dir(), 'images')
|
||||
img_root_dir = img_root_dir or os.path.join(uhdimgs.get_images_dir(), 'images')
|
||||
if not os.path.isdir(img_root_dir):
|
||||
print("== Creating images directory...")
|
||||
os.mkdir(img_root_dir)
|
||||
os.chdir(uhdimgs.get_images_dir())
|
||||
print "== Starting download..."
|
||||
print("== Starting download...")
|
||||
try:
|
||||
downloader_cmd = [
|
||||
'python',
|
||||
'../host/utils/uhd_images_downloader.py.in',
|
||||
'-i', img_root_dir,
|
||||
'-m', 'manifest.txt'
|
||||
'python',
|
||||
'../host/utils/uhd_images_downloader.py.in',
|
||||
'-i', img_root_dir,
|
||||
'-m', 'manifest.txt'
|
||||
]
|
||||
subprocess.check_call(downloader_cmd)
|
||||
except (subprocess.CalledProcessError, OSError):
|
||||
print "[ERROR] Failed to run downloader script."
|
||||
print("[ERROR] Failed to run downloader script.")
|
||||
exit(1)
|
||||
print "== Done!"
|
||||
print("== Done!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
download_images()
|
||||
|
|
|
|||
|
|
@ -1,24 +1,15 @@
|
|||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2014 Ettus Research LLC
|
||||
# Copyright 2018 Ettus Research, a National Instruments Company
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
#
|
||||
"""
|
||||
Utility module for packaging and handling UHD binary images.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -75,12 +66,12 @@ def get_total_md5(img_dir):
|
|||
for filename in filenames:
|
||||
md5_obj.update(open(os.path.join(root, filename), 'rb').read())
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.flus()
|
||||
sys.stdout.flush()
|
||||
for dirname in dirnames:
|
||||
_update_md5_for_dir_recursive(os.path.join(root, dirname), md5_obj)
|
||||
md5 = hashlib.md5()
|
||||
_update_md5_for_dir_recursive(img_dir, md5)
|
||||
print ""
|
||||
print("")
|
||||
return md5.hexdigest()
|
||||
|
||||
def md5_checksum(filePath):
|
||||
|
|
@ -94,6 +85,6 @@ def md5_checksum(filePath):
|
|||
break
|
||||
m.update(data)
|
||||
return m.hexdigest()
|
||||
except Exception, e:
|
||||
print "Failed to calculated MD5 sum of: %s (%s)" % (filePath, e)
|
||||
except Exception as e:
|
||||
print("Failed to calculated MD5 sum of: %s (%s)" % (filePath, e))
|
||||
raise e
|
||||
|
|
|
|||
Loading…
Reference in a new issue