[SOLVED] Can't Compile Python 3.6.1

OstermanA · 06-16-2017, 05:45 PM

First off. Compiling Python is a nightmare. The whole "We couldn't find all the bits to make the SSL module, go read the script we used to determine that and reverse engineer it to see if you can figure out why not!" is unbelievably useless as an error message.

That said. I'm getting something that's arcane rather than obtuse at this point, and I'm not sure how to read it.

Code:

     [exec] gcc -pthread -c -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -Wl,-rpath,/sites/opt/PYTHON/Python-3.6.1/lib,-rpath,/sites/opt/openssl/lib -I/sites/opt/PYTHON/Python-3.6.1/include -I/sites/opt/openssl/include -L/sites/opt/PYTHON/Python-3.6.1/lib -L/sites/opt/openssl/lib -fPIC   -std=c99 -Wextra -Wno-unused-result -Wno-unused-parameter -Wno-missing-field-initializers -fprofile-generate   -I. -I./Include -Wl,-rpath,/sites/opt/PYTHON/Python-3.6.1/lib,-rpath,/sites/opt/openssl/lib -I/sites/opt/PYTHON/Python-3.6.1/include -I/sites/opt/openssl/include -L/sites/opt/PYTHON/Python-3.6.1/lib -L/sites/opt/openssl/lib   -DPy_BUILD_CORE -o Python/thread.o Python/thread.c
     [exec] In file included from /usr/include/string.h:633:0,
     [exec]                  from ./Include/Python.h:30,
     [exec]                  from ./Include/pgenheaders.h:10,
     [exec]                  from Python/strdup.c:3:
     [exec] Python/strdup.c:6:1: error: expected identifier or ‘(’ before ‘__extension__’
     [exec]  strdup(const char *str)
     [exec]  ^
     [exec] make[2]: *** [Python/strdup.o] Error 1
     [exec] make[2]: *** Waiting for unfinished jobs....
     [exec] make[2]: Leaving directory `/sites/usr/src/Python-3.6.1'
     [exec] make[1]: Leaving directory `/sites/usr/src/Python-3.6.1'
     [exec] make[1]: *** [build_all_generate_profile] Error 2
     [exec] make: *** [profile-opt] Error 2

This is the Ant script I'm using to run the build:

Code:

<?xml version="1.0"?>
<project name="Python 3.6.1" default="main" basedir="..">
  <!-- Python properties -->
  <property name="Python.file" value="Python-3.6.1.tgz"/>
  <property name="Python.dir" value="Python-3.6.1"/>
  <!-- zlib properties -->
  <property name="zlib.file" value="zlib-1.2.11.tar.gz"/>
  <property name="zlib.dir" value="zlib-1.2.11"/>
  <!-- Project properties -->
  <property name="install.base" value="/sites/opt"/>
  <property name="install.dir" value="${install.base}/PYTHON/${Python.dir}"/>
  <property name="openssl.dir" value="${install.base}/openssl"/>
  <property name="cflags" value="-Wl,-rpath,${install.dir}/lib,-rpath,${openssl.dir}/lib -I${install.dir}/include -I${openssl.dir}/include -L${install.dir}/lib -L${openssl.dir}/lib"/>
  <property name="cppflags" value="${cflags}"/>
  <!-- DO NOT MODIFY ANYTHING BELOW HERE -->
  <target name="main">
    <antcall target="sanitytest"/>
    <antcall target="clean"/>
    <antcall target="buildzlib"/>
    <antcall target="buildPython"/>
    <antcall target="clean"/>
  </target>
  <target name="sanitytest">
    <available file="${Python.file}" property="Python.file.exists"/>
    <fail message="${Python.file} does not exist" unless="Python.file.exists"/>
  </target>
  <target name="clean">
    <echo message="Removing build artifacts" level="info"/>
    <delete dir="${Python.dir}"/>
    <delete dir="${zlib.dir}"/>
  </target>
  <target name="buildPython">
    <echo message="Building ${Python.dir}" level="info"/>
    <echo message="Unpacking ${Python.file}" level="info"/>
    <exec executable="tar" failonerror="true">
      <arg value="-xf"/>
      <arg value="${Python.file}"/>
    </exec>
    <echo message="Enabling SSL" level="info"/>
    <exec executable="sed" dir="${Python.dir}" failonerror="true">
      <arg value="-i"/>
      <arg value="-r"/>
      <arg value="-e"/>
      <arg value="s@#SSL=/usr/local/ssl@SSL=/sites/opt/openssl@"/>
      <arg value="-e"/>
      <arg value="s@#([[:space:]]+-.*SSL)@\1@"/>
      <arg value="Modules/Setup.dist"/>
    </exec>
    <echo message="Configuring ${Python.dir}" level="info"/>
    <exec executable="bash" dir="${Python.dir}" failonerror="true">
      <env key="CFLAGS" value="${cflags} -fPIC"/>
      <env key="CPPFLAGS" value="${cppflags}"/>
      <env key="LD_LIBRARY_PATH" value="${install.dir}/lib:${openssl.dir}/lib"/>
      <arg value="configure"/>
      <arg value="--prefix=${install.dir}"/>
      <arg value="--enable-optimizations"/>
      <arg value="--disable-ipv6"/>
      <arg value="--with-threads"/>
      <arg value="--with-libs='${openssl.dir}/lib/libssl.so.1.0.0'"/>
    </exec>
    <echo message="Running 'make'" level="info"/>
    <exec executable="make" dir="${Python.dir}" failonerror="true">
      <arg value="-j6"/>
    </exec>
    <echo message="Running 'make install'" level="info"/>
    <exec executable="make" dir="${Python.dir}" failonerror="true">
      <arg value="install"/>
    </exec>
  </target>
  <target name="buildzlib">
    <echo message="Building ${zlib.dir}" level="info"/>
    <echo message="Unpacking ${zlib.file}" level="info"/>
    <exec executable="tar" failonerror="true">
      <arg value="-xf"/>
      <arg value="${zlib.file}"/>
    </exec>
    <echo message="Configuring ${zlib.dir}" level="info"/>
    <exec executable="bash" dir="${zlib.dir}" failonerror="true">
      <env key="CFLAGS" value="${cflags} -fPIC"/>
      <env key="CPPFLAGS" value="${cppflags}"/>
      <arg value="configure"/>
      <arg value="--prefix=${install.dir}"/>
      <arg value="--64"/>
      <arg value="--shared"/>
      <arg value="--static"/>
    </exec>
    <echo message="Running 'make'" level="info"/>
    <exec executable="make" dir="${zlib.dir}" failonerror="true">
      <arg value="-j6"/>
    </exec>
    <echo message="Running 'make check'" level="info"/>
    <exec executable="make" dir="${zlib.dir}" failonerror="true">
      <arg value="check"/>
    </exec>
    <echo message="Running 'make install'" level="info"/>
    <exec executable="make" dir="${zlib.dir}" failonerror="true">
      <arg value="install"/>
    </exec>
  </target>
</project>

It's a bit of a mess at the moment, as I'm still in the "throw things until it sticks" phase. I can clean it up when it actual builds.

Myk267 · 06-16-2017, 08:53 PM

The curiosity is going to kill me but... Why are you using Ant to compile Python?

I just compiled it by the usual incantation:

Code:

./configure && make

And everything went as expected. But I'm betting that you don't want to do this for some exotic reason you have yet to let on. So please do.

norobro · 06-16-2017, 10:22 PM

Try compiling without the "-O3" optimization option. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59668

OstermanA · 06-16-2017, 11:34 PM

Quote:

Originally Posted by Myk267

The curiosity is going to kill me but... Why are you using Ant to compile Python?

I just compiled it by the usual incantation:

Code:

./configure && make

And everything went as expected. But I'm betting that you don't want to do this for some exotic reason you have yet to let on. So please do.

I'm building this out for servers I don't have root on. I have to compile it in a non-standard location, and my system admins are idiots so I don't want to depend on their software more than I absolutely have to. I'm essentially building an entire toolchain that I control so I don't get completely screwed the next time they decide to change things without telling anyone. Ant got involved because I wanted my build options to be recorded under version control and Ant is what I knew. I started in Bash, but got really tired of trying to trap all of the errors, when a build control scripting language was so readily available.

As for disabling -03: That's being put in by the Python make process. While continuing to fight with this, I've removed the '--enable-optimizations' with no effect. So... I'm not sure what else to do. Just for fun, I tried putting in '--disable-optimizations' but it did exactly nothing, as expected.

OstermanA · 06-18-2017, 10:13 PM

OK. I found the root cause of all the bizarre errors I've been facing.

Python REALLY DOES NOT LIKE when you run the make file in multiple threads. 'make -j6' is a giant bundle of fail, but a plan old 'make' works just fine.

It would appear that when divvying tasks up between the threads, some things get built too soon and fail with effectively untraceable errors. There's a lot of optimization left for my "preferred" solution, but I think I can work with this.

Edit:
It also seems that python does not like linking to external libraries, and wants them on the same prefix. It's possible that it can be made to do so, but if so it's beyond my skill level. Meh. Having a few extra copies of openssl, et al, is hardly a catastrophe. At least the thing works.

Have yet to add bz2/lzma/readline support (mostly because I can), but this is the current state of the Ant build:

Code:

<?xml version="1.0"?>
<project name="Python 3.6.1" default="main" basedir="..">
  <!-- openssl properties -->
  <property name="openssl.file" value="openssl-1.0.2l.tar.gz"/>
  <property name="openssl.dir" value="openssl-1.0.2l"/>
  <!-- Python properties -->
  <property name="Python.file" value="Python-3.6.1.tgz"/>
  <property name="Python.dir" value="Python-3.6.1"/>
  <!-- sqlite properties -->
  <property name="sqlite.file" value="sqlite-autoconf-3190300.tar.gz"/>
  <property name="sqlite.dir" value="sqlite-autoconf-3190300"/>
  <!-- zlib properties -->
  <property name="zlib.file" value="zlib-1.2.11.tar.gz"/>
  <property name="zlib.dir" value="zlib-1.2.11"/>
  <!-- Project properties -->
  <property name="install.base" value="/sites/opt"/>
  <property name="install.dir" value="${install.base}/PYTHON/${Python.dir}"/>
  <property name="cflags" value="-Wl,-rpath,${install.dir}/lib"/>
  <property name="cppflags" value="${cflags}"/>
  <!-- DO NOT MODIFY ANYTHING BELOW HERE -->
  <target name="main">
    <antcall target="sanitytest"/>
    <antcall target="clean"/>
    <antcall target="buildzlib"/>
    <antcall target="buildsqlite"/>
    <antcall target="buildopenssl"/>
    <antcall target="buildPython"/>
    <antcall target="clean"/>
  </target>
  <target name="sanitytest">
    <available file="${openssl.file}" property="openssl.file.exists"/>
    <available file="${Python.file}" property="Python.file.exists"/>
    <available file="${sqlite.file}" property="zlib.file.exists"/>
    <available file="${zlib.file}" property="zlib.file.exists"/>
    <fail message="${openssl.file} does not exist" unless="openssl.file.exists"/>
    <fail message="${Python.file} does not exist" unless="Python.file.exists"/>
    <fail message="${sqlite.file} does not exist" unless="zlib.file.exists"/>
    <fail message="${zlib.file} does not exist" unless="zlib.file.exists"/>
  </target>
  <target name="clean">
    <echo message="Removing build artifacts" level="info"/>
    <delete dir="${Python.dir}"/>
    <delete dir="${openssl.dir}"/>
    <delete dir="${sqlite.dir}"/>
    <delete dir="${zlib.dir}"/>
  </target>
  <target name="buildPython">
    <echo message="Building ${Python.dir}" level="info"/>
    <echo message="Unpacking ${Python.file}" level="info"/>
    <exec executable="tar" failonerror="true">
      <arg value="-xf"/>
      <arg value="${Python.file}"/>
    </exec>
    <echo message="Configuring ${Python.dir}" level="info"/>
    <exec executable="bash" dir="${Python.dir}" failonerror="true">
      <env key="CFLAGS" value="${cflags}"/>
      <env key="CPPFLAGS" value="${cppflags}"/>
      <arg value="configure"/>
      <arg value="--prefix=${install.dir}"/>
      <arg value="--enable-optimizations"/>
    </exec>
    <echo message="Running 'make'" level="info"/>
    <exec executable="make" dir="${Python.dir}" failonerror="true">
    </exec>
    <echo message="Running 'make install'" level="info"/>
    <exec executable="make" dir="${Python.dir}" failonerror="true">
      <arg value="install"/>
    </exec>
  </target>
  <target name="buildopenssl">
    <echo message="Building ${openssl.dir}" level="info"/>
    <echo message="Unpacking ${openssl.file}" level="info"/>
    <exec executable="tar" failonerror="true">
      <arg value="-xf"/>
      <arg value="${openssl.file}"/>
    </exec>
    <echo message="Configuring ${openssl.dir}" level="info"/>
    <exec executable="bash" dir="${openssl.dir}" failonerror="true">
      <env key="CFLAGS" value="${cflags}"/>
      <env key="CPPFLAGS" value="${cppflags}"/>
      <arg value="config"/>
      <arg value="--prefix=${install.dir}"/>
      <arg value="--openssldir=${install.dir}"/>
      <arg value="threads"/>
      <arg value="shared"/>
      <arg value="no-comp"/>
      <arg value="${cflags}"/>
    </exec>
    <echo message="Running 'make depend'" level="info"/>
    <exec executable="make" dir="${openssl.dir}" failonerror="true">
      <arg value="depend"/>
    </exec>
    <echo message="Running 'make'" level="info"/>
    <exec executable="make" dir="${openssl.dir}" failonerror="true">
      <arg value="-j6"/>
    </exec>
    <echo message="Running 'make test'" level="info"/>
    <exec executable="make" dir="${openssl.dir}" failonerror="true">
      <arg value="test"/>
    </exec>
    <echo message="Running 'make install'" level="info"/>
    <exec executable="make" dir="${openssl.dir}" failonerror="true">
      <arg value="install"/>
    </exec>
  </target>
  <target name="buildsqlite">
    <echo message="Building ${sqlite.dir}" level="info"/>
    <echo message="Unpacking ${sqlite.file}" level="info"/>
    <exec executable="tar" failonerror="true">
      <arg value="-xf"/>
      <arg value="${sqlite.file}"/>
    </exec>
    <echo message="Configuring ${sqlite.dir}" level="info"/>
    <exec executable="bash" dir="${sqlite.dir}" failonerror="true">
      <env key="CFLAGS" value="${cflags}"/>
      <env key="CPPFLAGS" value="${cppflags}"/>
      <arg value="configure"/>
      <arg value="--prefix=${install.dir}"/>
    </exec>
    <echo message="Running 'make'" level="info"/>
    <exec executable="make" dir="${sqlite.dir}" failonerror="true">
      <arg value="-j6"/>
    </exec>
    <echo message="Running 'make install'" level="info"/>
    <exec executable="make" dir="${sqlite.dir}" failonerror="true">
      <arg value="install"/>
    </exec>
  </target>
  <target name="buildzlib">
    <echo message="Building ${zlib.dir}" level="info"/>
    <echo message="Unpacking ${zlib.file}" level="info"/>
    <exec executable="tar" failonerror="true">
      <arg value="-xf"/>
      <arg value="${zlib.file}"/>
    </exec>
    <echo message="Configuring ${zlib.dir}" level="info"/>
    <exec executable="bash" dir="${zlib.dir}" failonerror="true">
      <env key="CFLAGS" value="${cflags} -fPIC"/>
      <env key="CPPFLAGS" value="${cppflags}"/>
      <arg value="configure"/>
      <arg value="--prefix=${install.dir}"/>
      <arg value="--64"/>
      <arg value="--shared"/>
      <arg value="--static"/>
    </exec>
    <echo message="Running 'make'" level="info"/>
    <exec executable="make" dir="${zlib.dir}" failonerror="true">
      <arg value="-j6"/>
    </exec>
    <echo message="Running 'make check'" level="info"/>
    <exec executable="make" dir="${zlib.dir}" failonerror="true">
      <arg value="check"/>
    </exec>
    <echo message="Running 'make install'" level="info"/>
    <exec executable="make" dir="${zlib.dir}" failonerror="true">
      <arg value="install"/>
    </exec>
  </target>
</project>