#!/bin/bash set -eu set -o pipefail # set -x export which_pythons="" export cleanup=True export do_gtk=True export do_numerical=True export macos_openssl=openssl@3 # Turn up automake verbosity export V=1 # Turn up cmake verbosity export VERBOSE=1 initial_dir="$(pwd)" PATH="$initial_dir/bin:$PATH" export PATH function usage { retval="$1" ( echo "Usage: $0 --which-pythons 'python-2.7 python-3.4'" echo " $0 --which-pythons all" echo " --skip-gtk" echo " --skip-numerical" echo " --skip-cleanup do not remove build temporaries" ) 1>&2 exit "$retval" } while [ "$#" -ge 1 ] do case "$1" in --which-pythons) which_pythons="$2" shift ;; --skip-gtk) do_gtk=False ;; --skip-numerical) do_numerical=False ;; --skip-cleanup) cleanup=False ;; -h|--help) usage 0 ;; *) echo "$0: Unrecognized option: $1" 1>&2 usage 1 ;; esac shift done if [ "$(whoami)" != root ] then echo "$0: You must run me as root" 1>&2 exit 1 fi ( cd cobble make install ) # shellcheck disable=SC2230 if ! which cobble > /dev/null 2>&1 then echo "$0: I require cobble. You can get cobble from http://stromberg.dnsalias.org/~strombrg/cobble.html" 1>&2 exit 1 fi if [ "${which_pythons:-}" = "" ] then echo "$0: --which-pythons is a required option" 1>&2 echo "Give it 'all' or a space-separated list like 'python-2.7 python-3.4'" usage 1 fi # shellcheck disable=SC2155 export original_cwd=$(set -eu; set -o pipefail; pwd) function all_pythons { echo python-3.11 echo python-3.10 echo python-3.9 echo python-3.8 echo python-3.7 echo python-3.6 echo python-3.5 echo python-3.4 echo python-3.3 echo python-3.2 echo python-3.1 echo python-3.0 echo python-2.7 echo python-2.6 echo python-2.5 echo python-2.4 echo python-2.3 echo python-2.2 echo python-2.1 echo python-2.0 echo python-1.6 echo python-1.5 echo python-1.4 echo python-1.3 echo python-1.2 echo python-1.1 echo python-1.0 echo python-0.9 } case "$which_pythons" in all) which_pythons=$(all_pythons) ;; *) ;; esac # Intentionally not quoting $which_pythons for python in $which_pythons do case "$python" in c*) echo "$0: $python starts with a c, but must not" 1>&2 usage 1 ;; esac case "$python" in *-*) ;; *) echo "$0: $python does not contain a -, but must" 1>&2 usage 1 ;; esac done function getlogin { python3 -c ' import os print(os.getlogin()) ' } if type apt-get > /dev/null 2>&1 then # gcc-multilib is to get a 32 bit build environment, since some of the older pythons segfault sometimes when # built for 64 bit. # Note that meson from apt-get was too old. I had to build one myself. It's still listed here, hoping # one day that won't be necessary. # libbonobo2-dev \ apt-get -y install \ autoconf \ automake \ autopoint \ bison \ build-essential \ clang \ cmake \ flex \ g++ \ gcc-multilib \ gfortran \ gir1.2-gtk-4.0 \ git \ gperf \ gtk-doc-tools \ intltool \ libbz2-dev \ libcairo2-dev \ libdb-dev \ libdbus-glib-1-dev \ libegl1-mesa-dev \ libepoxy-dev \ libexpat1-dev \ libffi-dev \ libfribidi-dev \ libgconf2-dev \ libgdbm-dev \ libgirepository1.0-dev \ liblzma-dev \ libmount-dev \ libpng-dev \ libreadline-dev \ libsasl2-dev \ libsqlite3-dev \ libssl-dev \ libtool \ libwayland-dev \ libxext-dev \ libxinerama-dev \ libxkbcommon-dev \ libxml2-utils \ libxrandr-dev \ libxtst-dev \ mesa-common-dev \ meson \ pkg-config \ python3-dev \ python3-setuptools \ ragel \ sqlite3 \ swig # This is no longer available nor required in Mint 18 apt-get install libXi-dev || true have_ragel=True have_xinput=True elif type yum > /dev/null 2>&1 then # Worked with CentOS 7. yum -y groupinstall "Development Tools" yum -y install \ openssl-devel \ libffi-devel \ cmake \ expat-devel \ libpng-devel \ dbus-glib-devel \ libXtst-devel \ bzip2-devel \ xz-devel \ ncurses-devel \ wget \ libXi-devel \ libmount-devel \ colm-devel \ libwayland-devel have_ragel=False have_xinput=False elif type brew > /dev/null 2>&1 then su "$(getlogin)" -c "brew install $macos_openssl zlib readline xz meson" # cp /usr/local/opt/openssl@1.1/lib/pkgconfig/*.pc /usr/local/lib/pkgconfig/. # cp /usr/local/opt/zlib/lib/pkgconfig/*.pc /usr/local/lib/pkgconfig/. else echo "$0: None of apt-get, yum or brew found" 1>&2 exit 1 fi #function get_includes #( # find /usr/local/$which_python/include -type d -print | sed 's/^/-I/' | tr '\012' ' ' #) function less_than { x="$1" y="$2" if echo "$x"-"$y" | bc | grep -Eq '^-' then return 0 else return 1 fi } function autoconf_downrev { autoconf_version=$(autoconf --version | head -1 | awk ' { print $NF }') if less_than "$autoconf_version" 2.64 then return 0 else return 1 fi } function pygi ( set -eu set -o pipefail cd "$initial_dir"/pygi/meson cobble do-inst # This seems to help pkg-config find the newer files. cd "$initial_dir"/pygi/pkg-config cobble do-inst export PKG_CONFIG="/usr/local/c$which_python/bin/pkg-config" PKG_CONFIG_PATH="$(set -eu; build_pkg_config_path)" export PKG_CONFIG_PATH export PATH=/usr/local/c"$which_python"/bin:$PATH # build and install glib cd "$initial_dir"/pygi/glib cobble do-inst # build and install gobject-introspection cd "$initial_dir"/pygi/gobject-introspection cobble do-inst # According to: # https://www.linuxquestions.org/questions/linux-from-scratch-13/make-%2A%2A%2A-%5B-sources-blfs-freetype-2-6-3-objs-autofit-lo%5D-error-1-a-4175583724/ # ...we need to install freetype twice, once before harfbuzz, once after. # build and install freetype2, seems to be needed by harfbuzz cd "$initial_dir"/pygi/freetype2 cobble do-inst # This too appears to be included in GTK+ now - but do we benefit from it? # On zareason-strata7440, I have it in /usr/local/lib, but on dell-inspiron-3530 I do not. # And the build succeeds on zs7 but not di3. # build and install harfbuzz, cairo seems to need it cd "$initial_dir"/pygi/harfbuzz cobble do-inst # build and install freetype2, seems to be needed by cairo cd "$initial_dir"/pygi/freetype2 cobble do-inst # build and install atk # GTK+ uses this, and it has a typelib, so we probably need it to come after gobject-introspection # Also, pixman uses it, and we do not want pixman trying to get the OS-provided ATK cd "$initial_dir"/pygi/atk cobble do-inst # build and install pixman, seems to be needed by cairo (?) cd "$initial_dir"/pygi/pixman cobble do-inst # build and install fontconfig, building this after cairo gave an unresolved external # cairo_ft_font_options_substitute cd "$initial_dir"/pygi/fontconfig cobble do-inst # build and install cairo # pango seems to need cairo, so we build cairo before pango cd "$initial_dir"/pygi/cairo cobble do-inst # build and install pango # This was griping about not having gobject-introspection... # Pango appears to be included in GTK+'s source distribution now; let's use that. cd "$initial_dir"/pygi/pango cobble do-inst # We redo this, because pango appears to be putting things in a different spot PKG_CONFIG_PATH="$(set -eu; build_pkg_config_path)" export PKG_CONFIG_PATH case "$have_ragel" in True) ;; False) # build and install ragel; we were unable to get it from a package manager cd "$initial_dir"/pygi/ragel cobble do-inst ;; *) echo "$0: Internal error: \$have_ragel is neither True nor False" 1>&2 exit 1 ;; esac case "$have_xinput" in True) ;; False) # build and install ragel; we were unable to get it from a package manager cd "$initial_dir"/pygi/xinput cobble do-inst ;; *) echo "$0: Internal error: \$have_xinput is neither True nor False" 1>&2 exit 1 ;; esac # If autoconf is too old, install that too # Actually, autoconf hasn't changed in years, but I want to try building it into the cpython hierarchy # to see if that'll help with a "error: HAVE_INTROSPECTION does not appear in AM_CONDITIONAL" I am # getting. # if autoconf_downrev # then cd "$initial_dir"/pygi/autoconf cobble do-inst # fi # build and install libtiff cd "$initial_dir"/pygi/libtiff cobble do-inst # build and install libjpeg cd "$initial_dir"/pygi/libjpeg cobble do-inst # build and install gdk-pixbuf cd "$initial_dir"/pygi/gdk-pixbuf cobble do-inst # Build and install at-spi - GTK+ requires this. We need it for # atk-bridge, perhaps more. Oddly, this requires GTK+-2.0! # cd "$initial_dir"/pygi/at-spi # cobble # do-inst # Build and install at-spi2-core cd "$initial_dir"/pygi/at-spi2-core cobble do-inst # Build and install spi2-atk - GTK+ requires this (?) for atk-bridge (?) cd "$initial_dir"/pygi/at-spi2-atk cobble do-inst # Build and install graphene - GTK+ appears to require this cd "$initial_dir"/pygi/graphene cobble do-inst # We redo this, because pango appears to be putting things in a different spot PKG_CONFIG_PATH="$(set -eu; build_pkg_config_path)" export PKG_CONFIG_PATH # build and install gtk+ cd "$initial_dir"/pygi/gtk+ cobble do-inst # This seems to want Gnome, which is more than I want to build # Build and install bonobo #cd "$initial_dir"/pygi/bonobo2 #cobble #do-inst # Build and install pycairo. # Apparently these folks are going dual-codebase. case "$which_python" in python-2.*) # We no longer build py2cairo for python 2.x. ;; python-3.*) cd "$initial_dir"/pygi/pycairo ;; *) echo "Neither 2.x nor 3.x. Has a lot of time passed, or is \$which_python misdefined?" exit 1 ;; esac cobble 'do-inst' # build and install pygi cd "$initial_dir"/pygi cobble do-inst # Clean up case "$cleanup" in True) rm -rf "$initial_dir"/pygi/*/src rm -rf "$initial_dir"/pygi/src ;; False) ;; *) echo "$0: Internal error 1: Invalid \$cleanup: $cleanup" 1>&2 exit 1 ;; esac ) function pygobject_via_pip { apt install 'libgirepository1.0-dev' 'gcc' 'libcairo2-dev' 'pkg-config' 'python3-dev' 'gir1.2-gtk-4.0' "$PIP" install pycairo "$PIP" install PyGObject } function install_setuptools { set -x # pylint needs setuptools case "$which_python" in python-0.*|python-1.*|python-2.[0123456]|python-3.[01234567]) # 3.[012] and python 2.[456] don't appear to have pip accessible via get-pip.py . # We pretend we have pip in a manner sometimes used successfully for ranlib. # # ensurepip segfaults (!) on CPython 3.7 now (2023-07-04) PIP=true export PIP return ;; *) # 3.5 come with a pip installer - use that # # 2.7.9 also has the pip installer: ensurepip /usr/local/c"$which_python"/bin/python -m ensurepip export PIP="/usr/local/c$which_python/bin/python -m pip" # pip install virtualenv ;; esac # Make sure we have the latest version eval "$PIP install --upgrade pip" } function install_pylint { case "$which_python" in python-3.[0123]|python-2.[0123456]|python-1.*|python-0.*) # pylint fails on cpython 3.[012] # pylint fails to install due to an isort issue in cpython 3.3 # recent pylint (1.0) fails to build on 2.[45] : ;; *) # setup_py_install_pylint pip_install_pylint ;; esac } function install_pygments { case "$which_python" in python-3.[0123456]|python-2.*|python-1.*|python-0.*) ;; *) eval "$PIP install pygments" ;; esac } function install_mypy { case "$which_python" in @python-3.10) # 3.10 is apparently too-new for mypy ;; python-3.[012]|python-2.*|python-1.*|python-0.*) # mypy is not available for older cpythons : ;; *) # mypy-lang was renamed to just "mypy". # $PIP install mypy-lang eval "$PIP install mypy" ;; esac } function install_six { # six is a compatibility module for python 2.x and 3.x case "$which_python" in python-2.[012345]|python-1.*|python-0.*) # six is not available for cpython 2.[45] : ;; *) eval "$PIP install six" ;; esac } function pip_install_pylint { case "$which_python" in python-2.7) # Pylint is moving on from Python 2.x, but the older versions still work: # https://stackoverflow.com/questions/51746255/how-can-i-install-the-pylint-for-python2-7 eval "$PIP install pylint==1.9.5" ;; *) eval "$PIP install pylint" ;; esac } function pip_install_flake8 { eval "$PIP install flake8" } function pip_install_pyflakes { eval "$PIP install pyflakes" } function pip_install_pycodestyle { eval "$PIP install pycodestyle" } function setup_py_install_pylint { case "$which_python" in python-2.5) # install stringformat for str.format functionality ( set -eu set -o pipefail cd pylint/stringformat cobble || true cd src /usr/local/c"$which_python"/bin/python setup.py install --no-compile ) ;; esac # install logilab common ( set -eu set -o pipefail cd pylint/common cobble || true cd src /usr/local/c"$which_python"/bin/python setup.py install --no-compile ) # install logilab astroid ( set -eu set -o pipefail cd pylint/astroid cobble || true cd src /usr/local/c"$which_python"/bin/python setup.py install --no-compile ) # install pylint proper ( set -eu set -o pipefail cd pylint cobble || true cd src /usr/local/c"$which_python"/bin/python setup.py install --no-compile ) permissions_workaround } function python3_symlink { # symlink python3 to python, if python3 present - for making scripts nice and consistent if ls /usr/local/c"$which_python"/bin/python3* > /dev/null 2>&1 then ( set -eu set -o pipefail cd /usr/local/c"$which_python"/bin # shellcheck disable=SC2010 first=$(ls -f | grep -E '^python3[0-9\.]*$' | sort | head -1) ln -sf "$first" python # I had a case statement for this, but vim did not like it if echo "$which_python" | grep -Eq '^python-3' then if ! [ -e python3 ] then ln -sf "$first" python3 fi fi if ! [ -e pip ] then ln -sf pip3 pip fi ) fi } function permissions_workaround { # a workaround for a permissions issue # /usr/local/cpython-2.5/lib/python2.5/site-packages/logilab/astng/brain # shellcheck disable=SC2001 sans_dash=$(set -eu; set -o pipefail; echo "$which_python" | sed -e 's/-//g') #sans_dash=${which_python//-/} dir="/usr/local/$which_python/lib/$sans_dash/site-packages/logilab/astng/brain" if [ -e "$dir" ] then chmod -v 644 "$dir"/* fi } function install_cython { # install cython case "$which_python" in @python-3.9) # 3.9 alpha 3 is too new for cython return ;; esac # shellcheck disable=SC2194 case 2 in 1) # This is the old way, with downloading, commiting and cobble'ing ( set -eu set -o pipefail cd cython cobble cd src # shellcheck disable=SC2030 export LDFLAGS="-L/usr/local/c$which_python/lib -Wl,-rpath -Wl,/usr/local/c$which_python/lib" /usr/local/c"$which_python"/bin/python setup.py build install ) ;; 2) # Cython doesn't appear to offer .tar.whatever downloads anymore. # So we get it from pypi. eval "$PIP install cython" ;; esac } function numerical_build { ( cd numerical directory=/usr/local/c"$which_python" export directory # shellcheck disable=SC2194 case 1 in 1) # build lapack using cmake ( cd lapack cobble cd src cmake '-DCMAKE_INSTALL_PREFIX:PATH'="$directory" "$(pwd)" make cd .. 'do-inst' rm -rf src ) ;; 2) # build lapack using make ( cd lapack # This errors out on the lack of a make.inc - 'local-script' creates one cobble || true cd src # This assumes 'local-script.nopers' is 'local-script' sh -x '../local-script' cd .. 'do-inst' rm -rf src ) ;; esac # Build a BLAS # shellcheck disable=SC2194 case openblas in openblas) # This uses traditional make, sans autoconf, sans cmake # This is a blas replacement, as well as a replacement for parts of lapack. # However, we tell it not to build any portions of lapack, instead relying on the real lapack, built above. ( cd openblas cobble 'do-inst' rm -rf src ) ;; atlas) # This wants to build lapack, but I'm attempting to tell it not to so we can build lapack with cmake # build blas and atlas # This may not be working :), but we're relying on openblas anyway. ( # atlas is a replacement for blas and parts of lapack cd atlas # This errors out about a make thing; we rely on that cobble || true 'do-inst' cd .. rm -rf src ) ;; *) echo "$0: Internal error: blas selection invalid" 1>&2 exit 1 esac # build suitesparse ( cd suitesparse tar xvfp SuiteSparse-*.tar.xz cd SuiteSparse #INSTALL_LIB = /usr/local/lib #INSTALL_INCLUDE = /usr/local/include FILE=SuiteSparse_config/SuiteSparse_config.mk sed -i "s#^INSTALL_LIB = .*\$#INSTALL_LIB = $directory/lib#" "$FILE" sed -i "s#^INSTALL_INCLUDE = .*\$#INSTALL_INCLUDE = $directory/include#" "$FILE" sed -i "s%^ BLAS = -lopenblas\$% BLAS = -L$directory/lib -Wl,-rpath -Wl,$directory/lib -lopenblas -lgfortran%" "$FILE" sed -i "s%^ LAPACK = -llapack\$% LAPACK = -L$directory/lib -Wl,-rpath -Wl,$directory/lib -llapack%" "$FILE" make make install cd .. rm -rf SuiteSparse ) # build numpy # shellcheck disable=SC2194 case "1" in 1) # This was working fine on 32 bit, but was having problems on 64 bit ( cd numpy # This fails, but we depend on that cobble || true cd src cp site.cfg.example site.cfg #library_dirs = /home/sam/local/lib #include_dirs = /home/sam/local/include # We change all directories, rather than trying to focus on "the right" ones #sed -i "s%^[# ]*library_dirs = .*\$%library_dirs = $directory/lib%" site.cfg #sed -i "s%^[# ]*include_dirs = .*\$%include_dirs = $directory/include%" site.cfg # there's related stuff in site.cfg.example (now site.cfg) for uncommenting, but appending like this is easier ( echo "[openblas]" echo "libraries = openblas" echo "library_dirs = $directory/lib" echo "include_dirs = $directory/include" ) >> site.cfg # there may be more to change in site.cfg. # http://www.scipy.org/scipylib/building/linux.html#building-everything-from-source-with-gfortran-on-ubuntu-nov-2010 # ...talks about uncommenting some things that no longer appear in site.cfg.example . "$directory"/bin/python setup.py build "$directory"/bin/python setup.py install --prefix="$directory" cd .. rm -rf src ) ;; 2) "$directory"/bin/pip install numpy ;; esac # build scipy ( cd scipy cobble || true cd src "$directory"/bin/python setup.py build "$directory"/bin/python setup.py install --prefix="$directory" cd .. rm -rf src ) ) case "$which_python" in 3.[678]) # Install tensorflow install_tensorflow ;; esac } function install_tensorflow { eval "$PIP install tensorflow" } # intentionally not quoting $which_pythons - we need to split on whitespace for which_python in $which_pythons do set -eu set -o pipefail export which_python # shellcheck disable=SC2001 which_python_sans_dash=$(echo "$which_python" | sed 's/-//') mkdir -p /usr/local/lib/"$which_python_sans_dash"/dist-packages/ export which_python_sans_dash # -Wformat works around a bug in CPython's autoconf case "$(uname -m)-$which_python" in i686-*) export CFLAGS=-Wformat ;; # CPython 1.4 doesn't appear to like this. *-python-1.[01235]|*-python-3.[0123]) # The older python's need to be built as a 32 bit executable. We could port to 64 bit, but that's # too much of a departure from history. # # CPython 3.3 needed this on Debian 11, so I assumed 3.[012] would as well. export CFLAGS="-m32 -fPIC -Wformat" export CXXFLAGS="-m32 -fPIC" export CPPFLAGS="$CXXFLAGS" export FFLAGS="-m32 -fPIC" export CC="gcc -m32 -fPIC" export CXX="g++ -m32 -fPIC" export FC="gfortran -m32 -fPIC" ;; x86_64-*) export CFLAGS="-m64 -fPIC -Wformat" export CXXFLAGS="-m64 -fPIC" export CPPFLAGS="$CXXFLAGS" export FFLAGS="-m64 -fPIC" export CC="gcc -m64 -fPIC" export CXX="g++ -m64 -fPIC" export FC="gfortran -m64 -fPIC" ;; *) echo "$0: Unrecognized architecture" 1>&2 exit 1 esac case "$(uname -s)" in Darwin) i=include CFLAGS="$CFLAGS -I$(brew --prefix $macos_openssl)/$i -I$(brew --prefix zlib)/$i -I$(brew --prefix readline)/$i" CPPFLAGS="$CFLAGS" LDFLAGS="$CFLAGS -L$(brew --prefix $macos_openssl)/lib -L$(brew --prefix zlib)/lib -L$(brew --prefix readline)/lib" export CFLAGS CPPFLAGS LDFLAGS ;; esac # This didn't help. # export CFLAGS="$CFLAGS -Wno-implicit-function-declaration" # shellcheck disable=SC2155,SC2001 export python_dir=$(set -eu; set -o pipefail; echo "$which_python" | sed -e 's/-//') #export python_dir=${which_python//-/} # clean up the old python rm -rf /usr/local/c"$which_python" ( # install python set -eu set -o pipefail cd "$python_dir" cobble 'do-inst' ) python3_symlink # symlink python3-config to python-config, if python3-config present - for making scripts nice and consistent if ls /usr/local/c"$which_python"/bin/python3-config > /dev/null 2>&1 then ( set -eu set -o pipefail cd /usr/local/c"$which_python"/bin first=python3-config ln -sf "$first" python-config ) fi # This has the side-effect of setting $PIP install_setuptools install_pygments install_pylint pip_install_flake8 pip_install_pyflakes pip_install_pycodestyle install_mypy install_six case "$which_python" in # python-2.[67]|python-3.[23456789]) python-3.[3456789]) install_cython ;; esac case "$do_gtk" in True) pygobject_via_pip || true ;; False) echo "$0: Skipping gtk by request of user" 1>&2 ;; *) echo "$0: Invalid value of \$do_gtk: $do_gtk" 1>&2 exit 1 ;; esac case "$do_numerical" in True) case "$which_python" in python-0.*|python-1.*|python-2.[01234567]|python-3.[0123456]) echo "$0: Skipping numerical build because this cpython is too old" 1>&2 ;; python-3.1[12]) echo "$0: Skipping numerical build because this cpython is too new" 1>&2 ;; python-3.*) # Actually, 3.1 appeared to build OK when I tried it, but the warning from openblas (?) says 2.6, 2.7 and >= 3.2 # We used to do this, but it was a big headache. Sun Dec 10 18:24:24 PST 2017 # numerical_build # So now, instead, we just pip install it eval "$PIP install numpy scipy" ;; *) echo "$0: Unrecognized cpython version" 1>&2 exit 1 esac ;; False) echo "$0: Skipping numerical by request of user" 1>&2 ;; *) echo "$0: Invalid value of \$do_numerical: $do_numerical" 1>&2 exit 1 ;; esac # Clean up the CPython and Cython source code and temporaries - we don't need them any longer, and they're not tiny. # Also clean up the last-archives files - they lead to too many merge conflicts. case "$cleanup" in True) rm -rf "$python_dir"/src rm -rf cython/src find . -name last-archives -print0 | xargs -0 rm -f ;; False) ;; *) echo "$0: Internal error 2: Invalid \$cleanup: $cleanup" 1>&2 exit 1 ;; esac done echo echo 'done'