Pyinstaller .exe cannot find _tiffile module - Loa

When I run my code from Pyinstaller the tiff reader works fine. After freezing using Pyinstaller I get the following warning:

UserWarning: ImportError: No module named '_tifffile'. Loading of some compressed images will be very slow. Tifffile.c can be obtained at http://www.lfd.uci.edu/~gohlke

And sure enough, a tiff file that used to take seconds to load into a numpy array may now take minutes.

Here is a simplified form of my code to focus on the problem. If you load an example tiff like this one it should load fast without problems.

If you use C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies tiffile_problems.py you should get a functional .exe with the above error message when you run it. When you try to load the same tiff it now takes much longer.

tiffile_problems.py

#!/usr/bin/env python3

import os
import sys
import traceback
import numpy as np
import matplotlib.pyplot as plt

from PyQt4.QtGui import *
from PyQt4.QtCore import *

sys.path.append('..')

from MBE_for_SO.util import fileloader, fileconverter

class NotConvertedError(Exception):
  pass

class FileAlreadyInProjectError(Exception):
  def __init__(self, filename):
    self.filename = filename

class Widget(QWidget):
  def __init__(self, project, parent=None):
    super(Widget, self).__init__(parent)

    if not project:
        self.setup_ui()
        return

  def setup_ui(self):
    vbox = QVBoxLayout()

    ## Related to importing Raws
    self.setWindowTitle('Import Raw File')

    #vbox.addWidget(QLabel('Set the size all data are to be rescaled to'))

    grid = QGridLayout()

    vbox.addLayout(grid)
    vbox.addStretch()

    self.setLayout(vbox)
    self.resize(400, 220)

    self.listview = QListView()
    self.listview.setStyleSheet('QListView::item { height: 26px; }')
    self.listview.setSelectionMode(QAbstractItemView.NoSelection)
    vbox.addWidget(self.listview)

    hbox = QVBoxLayout()
    pb = QPushButton('New Video')
    pb.clicked.connect(self.new_video)
    hbox.addWidget(pb)

    vbox.addLayout(hbox)
    vbox.addStretch()
    self.setLayout(vbox)


  def convert_tif(self, filename):
    path = os.path.splitext(os.path.basename(filename))[0] + '.npy'
    #path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting tif to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.tif2npy(filename, path, callback)
      print('Tifffile saved to wherever this script is')
    except:
      # qtutil.critical('Converting tiff to npy failed.')
      progress.close()
    return path

  def to_npy(self, filename):
    if filename.endswith('.raw'):
      print('No raws allowed')
      #filename = self.convert_raw(filename)
    elif filename.endswith('.tif'):
      filename = self.convert_tif(filename)
    else:
      raise fileloader.UnknownFileFormatError()
    return filename

  def import_file(self, filename):
    if not filename.endswith('.npy'):
      new_filename = self.to_npy(filename)
      if not new_filename:
        raise NotConvertedError()
      else:
        filename = new_filename

    return filename

  def import_files(self, filenames):
    for filename in filenames:
      try:
        filename = self.import_file(filename)
      except NotConvertedError:
        # qtutil.warning('Skipping file \'{}\' since not converted.'.format(filename))
        print('Skipping file \'{}\' since not converted.'.format(filename))
      except FileAlreadyInProjectError as e:
        # qtutil.warning('Skipping file \'{}\' since already in project.'.format(e.filename))
        print('Skipping file \'{}\' since already in project.'.format(e.filename))
      except:
        # qtutil.critical('Import of \'{}\' failed:\n'.format(filename) +\
        #   traceback.format_exc())
        print('Import of \'{}\' failed:\n'.format(filename) + traceback.format_exc())
      # else:
      #   self.listview.model().appendRow(QStandardItem(filename))

  def new_video(self):
    filenames = QFileDialog.getOpenFileNames(
      self, 'Load images', QSettings().value('last_load_data_path'),
      'Video files (*.npy *.tif *.raw)')
    if not filenames:
      return
    QSettings().setValue('last_load_data_path', os.path.dirname(filenames[0]))
    self.import_files(filenames)

class MyPlugin:
  def __init__(self, project):
    self.name = 'Import video files'
    self.widget = Widget(project)

  def run(self):
    pass

if __name__ == '__main__':
  app = QApplication(sys.argv)
  app.aboutToQuit.connect(app.deleteLater)
  w = QMainWindow()
  w.setCentralWidget(Widget(None))
  w.show()
  app.exec_()
  sys.exit()

fileconverter.py

#!/usr/bin/env python3

import os
import numpy as np

import tifffile as tiff

class ConvertError(Exception):
  pass

def tif2npy(filename_from, filename_to, progress_callback):
  with tiff.TiffFile(filename_from) as tif:
    w, h = tif[0].shape
    shape = len(tif), w, h
    np.save(filename_to, np.empty(shape, tif[0].dtype))
    fp = np.load(filename_to, mmap_mode='r+')
    for i, page in enumerate(tif):
      progress_callback(i / float(shape[0]-1))
      fp[i] = page.asarray()

def raw2npy(filename_from, filename_to, dtype, width, height,
  num_channels, channel, progress_callback):
    fp = np.memmap(filename_from, dtype, 'r')
    frame_size = width * height * num_channels
    if len(fp) % frame_size:
      raise ConvertError()
    num_frames = len(fp) / frame_size
    fp = np.memmap(filename_from, dtype, 'r',
      shape=(num_frames, width, height, num_channels))
    np.save(filename_to, np.empty((num_frames, width, height), dtype))
    fp_to = np.load(filename_to, mmap_mode='r+')
    for i, frame in enumerate(fp):
      progress_callback(i / float(len(fp)-1))
      fp_to[i] = frame[:,:,channel-1]

fileloader.py

#!/usr/bin/env python3

import numpy as np

class UnknownFileFormatError(Exception):
  pass

def load_npy(filename):
  frames = np.load(filename)
  # frames[np.isnan(frames)] = 0
  return frames

def load_file(filename):
  if filename.endswith('.npy'):
    frames = load_npy(filename)
  else:
    raise UnknownFileFormatError()
  return frames

def load_reference_frame_npy(filename, offset):
  frames_mmap = np.load(filename, mmap_mode='c')
  if frames_mmap is None:
    return None
  frame = np.array(frames_mmap[offset])
  frame[np.isnan(frame)] = 0
  frame = frame.swapaxes(0, 1)
  if frame.ndim == 2:
    frame = frame[:, ::-1]
  elif frame.ndim == 3:
    frame = frame[:, ::-1, :]
  return frame

def load_reference_frame(filename, offset=0):
  if filename.endswith('.npy'):
    frame = load_reference_frame_npy(filename, offset)
  else:
    raise UnknownFileFormatError()
  return frame

Why? And how do I fix this? I've located tifffile.py, tifffile.cpython-35.pyc, tifffile.c and placed them all in the same directory as the .exe. No effect. _tifffile.cp35-win_amd64.pyd is created by pyinstaller and placed in the same dir as the .exe. I don't know what other options are available to me.

tifffile_problems.spec

# -*- mode: python -*-

block_cipher = None


a = Analysis(['tiffile_problems.py'],
             pathex=['C:\\Users\\Cornelis\\PycharmProjects\\tester\\MBE_for_SO'],
             binaries=None,
             datas=None,
             hiddenimports=[],
             hookspath=[],
             runtime_hooks=[],
             excludes=[],
             win_no_prefer_redirects=False,
             win_private_assemblies=True,
             cipher=block_cipher)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)
exe = EXE(pyz,
          a.scripts,
          exclude_binaries=True,
          name='tiffile_problems',
          debug=False,
          strip=False,
          upx=True,
          console=True )
coll = COLLECT(exe,
               a.binaries,
               a.zipfiles,
               a.datas,
               strip=False,
               upx=True,
               name='tiffile_problems')

tiffile.spec when using C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies --onefile tiffile_problems.py

# -*- mode: python -*-

block_cipher = None


a = Analysis(['tiffile_problems.py'],
             pathex=['C:\\Users\\Cornelis\\PycharmProjects\\tester\\MBE_for_SO'],
             binaries=None,
             datas=None,
             hiddenimports=[],
             hookspath=['.'],
             runtime_hooks=[],
             excludes=[],
             win_no_prefer_redirects=False,
             win_private_assemblies=True,
             cipher=block_cipher)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)
exe = EXE(pyz,
          a.scripts,
          a.binaries,
          a.zipfiles,
          a.datas,
          name='tiffile_problems',
          debug=False,
          strip=False,
          upx=True,
          console=True )

回答1:

I think muggy is right about he weirdness with __package__ causing the issue here. I haven't tracked down the exact reason for the fix, but this seems to be resolved using the latest update to pyinstaller. Check your version with:

→ pyinstaller --version 3.2.1

and upgrade with

→ pip3 install --upgrade pyinstaller

The update was only made on January 15, 2017 so this wouldn't have helped when you originally asked, but it sure helps now.

回答2:

I actually seen this via upwork whilst I was just browsing around the net and decided to have a play around.

kazemakase was on the right track at the very start, when you run normally, the __package__ is None, and when its packaged __package__ is set to tifffile and the first condition is executed, and it becomes relative to the module tifffile.

if __package__:
    from . import _tifffile
else:
    import _tifffile

I just converted tifffile to a module manually, by creating in site-packages a tifffile folder, creating an empty __init__.py file in the new folder, placing tifffile.py, _tifffile.pyd from site-packages into the tifffile folder and changing the import statement, admittedly in a simple skeleton.

import tifffile.tifffile as tiff

If this helps across your whole project I don't know. And should note I used the wheel from http://www.lfd.uci.edu/~gohlke/pythonlibs/ to install originally to save the compilation step so your mileage may vary. I did the above initially on 2.7 but it also appears to work fine on 3.5 from some testing I was able to do. And didn't need to put anything in .spec files when I tested from the one originally generated.

回答3:

From inspecting the code, tifffile.py seems to be looking for a module called _tifffile, which presumably is the expected name of the compiled C extension:

try:
    if __package__:
        from . import _tifffile
    else:
        import _tifffile
except ImportError:
    warnings.warn(
        "ImportError: No module named '_tifffile'. "
        "Loading of some compressed images will be very slow. "
        "Tifffile.c can be obtained at http://www.lfd.uci.edu/~gohlke/")

The tifffile.cpython-35.pyc is just the bytecode generated from tiffile.py. You don't need to bother with that one.

The .c file alone won't help you, either. It needs to be compiled to to create a usable Python extension, which needs to be named something like _tiffile.cp35-win_amd64.pyd (can vary depending on your system and python version/installation) so it can be used by import _tifffile.

Compiling can be a daunting task if you have not done this before. If you feel you are up to it, the Python documentation can help you get started. You will need to have the same Compiler and settings that your Python version was compiled with.

However, there may be a simpler solution. If your code works fine before freezing, chances are you have the compiled extension correctly installed on your system. Pyinstaller probably misses it because it can find tifffile.py and is satisfied. Look for the correct .pyd file in your Python directories and see if you can modify the .spec file Pyinstaller created for your project, where you specify to include the .pyd file.

回答4:

Installation:

->If using conda, conda install tifffile -c conda-forge

-> Otherwise, pip install tifffile