#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
opsi-product-updater

opsi-product-updater is part of the desktop management solution opsi
(open pc server integration) http://www.opsi.org

Copyright (C) 2013-2015 uib GmbH

http://www.uib.de/

All rights reserved.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License, version 3
as published by the Free Software Foundation.

This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Affero General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.

@copyright:	uib GmbH <info@uib.de>
@author: Jan Schneider <j.schneider@uib.de>
@author: Erol Ueluekmen <e.ueluekmen@uib.de>
@author: Niko Wenselowski <n.wenselowski@uib.de>
@license: GNU Affero GPL version 3
"""

from __future__ import print_function

import formatter
import htmllib
import getopt
import os
import re
import smtplib
import socket
import sys
import time
import urllib
import urllib2

from OPSI.Logger import LOG_INFO, LOG_ERROR, Logger
from OPSI.Object import NetbootProduct, ProductOnClient
from OPSI.Types import (forceBool, forceEmailAddress, forceFilename,
	forceHostAddress, forceHostId, forceInt, forceProductId, forceUnicode,
	forceUnicodeList, forceUrl)
from OPSI.Util import compareVersions, md5sum, getfqdn
from OPSI.Util.File import IniFile, ZsyncFile
from OPSI.Util.Product import ProductPackageFile
from OPSI.Util.Task.Rights import setRights
from OPSI import System
from OPSI.Backend.BackendManager import BackendManager
from OPSI.Backend.JSONRPC import JSONRPCBackend

__version__ = '4.0.6.9'

logger = Logger()


def _(string):
	return string


# Patch from urllib2 HTTPBasicAuthHandler issue9639
# Only for Python >= 2.6.6
if sys.version_info[:2] == (2, 6) and sys.version_info[2] >= 6:
	def fixed_http_error_401(self, req, fp, code, msg, headers):
		url = req.get_full_url()
		response = self.http_error_auth_reqed('www-authenticate', url, req, headers)
		self.retried = 0
		return response

	urllib2.HTTPBasicAuthHandler.http_error_401 = fixed_http_error_401
# End of Patch


class ProductRepositoryInfo(object):
	def __init__(self, baseUrl, dirs=[], username=u"", password=u"", opsiDepotId=None, autoInstall=False, autoUpdate=True, autoSetup=False, proxy=None, excludes=[], includes=[]):
		self.baseUrl = forceUnicode(baseUrl)
		self.dirs = [forceUnicode(dir) for dir in dirs]
		self.excludes = excludes
		self.includes = includes
		self.username = forceUnicode(username)
		self.password = forceUnicode(password)
		self.autoInstall = autoInstall
		self.autoUpdate = autoUpdate
		self.autoSetup = autoSetup
		self.opsiDepotId = opsiDepotId
		self.onlyDownload = None
		self.inheritProductProperties = None

		self.proxy = None
		if proxy:
			self.proxy = proxy
		if self.baseUrl.startswith('webdav'):
			self.baseUrl = u'http%s' % self.baseUrl[6:]

	def getDownloadUrls(self):
		urls = []
		for directory in self.dirs:
			if directory in (u'', u'/', u'.'):
				url = self.baseUrl
			else:
				url = u'%s/%s' % (self.baseUrl, directory)

			if url not in urls:
				urls.append(url)
		return urls


class LinksExtractor(htmllib.HTMLParser):
	def __init__(self, formatter):
		htmllib.HTMLParser.__init__(self, formatter)
		self.links = []

	def start_a(self, attrs):
		if len(attrs) > 0:
			for attr in attrs:
				if attr[0] != "href":
					continue
				self.links.append(attr[1])

	def getLinks(self):
		return self.links


class EmailNotifier(object):
	def __init__(self, smtphost=u'localhost', smtpport=25, subject=u'opsi product updater', sender=u'', receivers=[]):
		self.receivers = forceUnicodeList(receivers)
		if not self.receivers:
			raise Exception(u"List of mail recipients empty")
		self.smtphost = forceUnicode(smtphost)
		self.smtpport = forceInt(smtpport)
		self.sender = forceUnicode(sender)
		self.subject = forceUnicode(subject)
		self.message = u''
		self.username = None
		self.password = None
		self.useStarttls = False

	def appendLine(self, line):
		now = unicode(time.strftime(u"%b %d %H:%M:%S", time.localtime()), 'utf-8', 'replace')
		self.message += u'%s %s\n' % (now, forceUnicode(line))

	def hasMessage(self):
		return bool(self.message)

	def notify(self):
		logger.notice(u"Sending mail notification")
		mail = u'From: %s\n' % self.sender
		mail += u'To: %s\n' % u','.join(self.receivers)
		mail += u'Subject: %s\n' % self.subject
		mail += u'\n'
		# mail += _(u"opsi product updater carried out the following actions:") + u"\n"
		mail += self.message
		smtpObj = None
		try:
			smtpObj = smtplib.SMTP(self.smtphost, self.smtpport)
			smtpObj.ehlo_or_helo_if_needed()

			if self.useStarttls:
				if smtpObj.has_extn('STARTTLS'):
					logger.debug('Enabling STARTTLS')
					smtpObj.starttls()
				else:
					logger.debug('Server does not support STARTTLS.')

			if self.username and self.password is not None:
				logger.debug(
					'Trying to authenticate against SMTP server '
					'{host}:{port} as user "{username}"'.format(
						host=self.smtphost,
						port=self.smtpport,
						username=self.username
					)
				)
				smtpObj.login(self.username, self.password)
				smtpObj.ehlo_or_helo_if_needed()

			smtpObj.sendmail(self.sender, self.receivers, mail)
			logger.debug(u"SMTP-Host: '%s' SMTP-Port: '%s'" % (self.smtphost, self.smtpport))
			logger.debug(u"Sender: '%s' Reveivers: '%s' Message: '%s'" % (self.sender, self.receivers, mail))
			logger.notice(u"Email successfully sent")
			smtpObj.quit()
		except Exception as error:
			if smtpObj is not None:
				logger.debug('SMTP Server does esmtp: {0}'.format(smtpObj.does_esmtp))
				if hasattr(smtpObj, 'ehlo_resp'):
					logger.debug('SMTP EHLO response: {0}'.format(smtpObj.ehlo_resp))

				if hasattr(smtpObj, 'esmtp_features'):
					logger.debug('ESMTP Features: {0}'.format(smtpObj.esmtp_features))

			raise Exception(u"Failed to send email using smtp server '%s': %s" % (self.smtphost, error))


class OpsiPackageUpdater(object):
	def __init__(self, config):
		self.config = config
		self.httpHeaders = {'User-Agent': self.config.get("userAgent", "")}
		self.configBackend = None
		self.depotConnections = {}
		self.depotId = forceHostId(getfqdn(conf='/etc/opsi/global.conf').lower())
		depots = self.getConfigBackend().host_getObjects(type='OpsiDepotserver', id=self.depotId)
		if not depots:
			raise Exception(u"Depot '%s' not found in backend" % self.depotId)
		self.depotKey = depots[0].opsiHostKey
		if not self.depotKey:
			raise Exception(u"Opsi host key for depot '%s' not found in backend" % self.depotId)
		logger.addConfidentialString(self.depotKey)
		self.readConfigFile()

	def __del__(self):
		for con in self.depotConnections.values():
			try:
				con.backend_exit()
			except Exception:
				pass

		if self.configBackend:
			try:
				self.configBackend.backend_exit()
			except Exception:
				pass

	def readConfigFile(self):
		try:
			logger.notice(u"Reading config file '%s'" % self.config["configFile"])
			if not os.path.isfile(self.config["configFile"]):
				raise Exception(u"File not found")

			self.config['repositories'] = []

			iniFile = IniFile(filename=self.config['configFile'], raw=True)
			config = iniFile.parse()
			for section in config.sections():
				if section.lower() == 'general':
					for (option, value) in config.items(section):
						if option.lower() == 'packagedir':
							self.config["packageDir"] = forceFilename(value.strip())
						elif option.lower() == 'logfile':
							value = forceFilename(value.strip())
							logger.setLogFile(value)
						elif option.lower() == 'loglevel':
							logger.setFileLevel(forceInt(value.strip()))
						elif option.lower() == 'timeout':
							socket.setdefaulttimeout(float(value.strip()))
						elif option.lower() == 'tempdir':
							self.config["tempdir"] = value.strip()

				elif section.lower() == 'notification':
					for (option, value) in config.items(section):
						if option.lower() == 'active':
							self.config["notification"] = forceBool(value)
						elif option.lower() == 'smtphost':
							self.config["smtphost"] = forceHostAddress(value.strip())
						elif option.lower() == 'smtpport':
							self.config["smtpport"] = forceInt(value.strip())
						elif option.lower() == 'smtpuser':
							self.config["smtpuser"] = forceUnicode(value.strip())
						elif option.lower() == 'smtppassword':
							self.config["smtppassword"] = forceUnicode(value.strip())
						elif option.lower() == 'subject':
							self.config["subject"] = forceUnicode(value.strip())
						elif option.lower() == 'use_starttls':
							self.config["use_starttls"] = forceBool(value.strip())
						elif option.lower() == 'sender':
							self.config["sender"] = forceEmailAddress(value.strip())
						elif option.lower() == 'receivers':
							self.config["receivers"] = []
							receivers = value.split(u",")
							for receiver in receivers:
								receiver = receiver.strip()
								if not receiver:
									continue
								self.config["receivers"].append(forceEmailAddress(receiver))

				elif section.lower() == 'wol':
					for (option, value) in config.items(section):
						if option.lower() == 'active':
							self.config["wolAction"] = forceBool(value.strip())
						elif option.lower() == 'excludeproductids':
							self.config['wolActionExcludeProductIds'] = []
							productIds = value.split(u',')
							for productId in productIds:
								productId = productId.strip()
								if not productId:
									continue
								self.config["wolActionExcludeProductIds"].append(forceProductId(productId))
						elif option.lower() == 'shutdownwanted':
							self.config["wolShutdownWanted"] = forceBool(value.strip())
						elif option.lower() == 'startgap':
							self.config["wolStartGap"] = forceInt(value.strip())
							if self.config["wolStartGap"] < 0:
								self.config["wolStartGap"] = 0

				elif section.lower() == 'installation':
					for (option, value) in config.items(section):
						if option.lower() == 'windowstart':
							if not value.strip():
								continue
							if not re.search('^\d{1,2}\:\d{1,2}$', value.strip()):
								raise Exception(u"Start time '%s' not in needed format 'HH:MM'" % value.strip())
							self.config["installationWindowStartTime"] = value.strip()
						elif option.lower() == 'windowend':
							if not value.strip():
								continue
							if not re.search('^\d{1,2}\:\d{1,2}$', value.strip()):
								raise Exception(u"End time '%s' not in needed format 'HH:MM'" % value.strip())
							self.config["installationWindowEndTime"] = value.strip()
						elif option.lower() == 'exceptproductids':
							self.config['installationWindowExceptions'] = []
							productIds = value.split(',')
							for productId in productIds:
								productId = productId.strip()
								if not productId:
									continue
								self.config["installationWindowExceptions"].append(forceProductId(productId))

				elif section.lower().startswith('repository'):
					try:
						if not forceBool(config.get(section, 'active')):
							continue
					except Exception:
						pass
					active = True
					baseUrl = None
					opsiDepotId = None
					proxy = None
					for (option, value) in config.items(section):
						if option.lower() == 'active':
							active = forceBool(value)
						elif option.lower() == 'baseurl':
							if value.strip():
								baseUrl = forceUrl(value.strip())
						elif option.lower() == 'opsidepotid':
							if value.strip():
								opsiDepotId = forceHostId(value.strip())
						elif option.lower() == 'proxy':
							if value.strip():
								proxy = forceUrl(value.strip())
					if not active:
						logger.info(u"Repository %s deactivated" % section)
						continue

					repository = None
					if opsiDepotId:
						depots = self.getConfigBackend().host_getObjects(type='OpsiDepotserver', id=opsiDepotId)
						if not depots:
							raise Exception(u"Depot '%s' not found in backend" % opsiDepotId)
						if not depots[0].repositoryRemoteUrl:
							raise Exception(u"Repository remote url for depot '%s' not found in backend" % opsiDepotId)
						repository = ProductRepositoryInfo(
							baseUrl=depots[0].repositoryRemoteUrl,
							dirs=['/'],
							username=self.depotId,
							password=self.depotKey,
							opsiDepotId=opsiDepotId
						)

					elif baseUrl:
						if proxy:
							logger.notice(u"Using Proxy: %s" % proxy)
						repository = ProductRepositoryInfo(baseUrl=baseUrl, proxy=proxy)
					else:
						logger.error(u"Repository section '%s': neither baseUrl nor opsiDepotId set" % section)
						continue

					for (option, value) in config.items(section):
						if option.lower() == 'username':
							repository.username = forceUnicode(value.strip())
						elif option.lower() == 'password':
							repository.password = forceUnicode(value.strip())
							if repository.password:
								logger.addConfidentialString(repository.password)
						elif option.lower() == 'autoinstall':
							repository.autoInstall = forceBool(value.strip())
						elif option.lower() == 'autoupdate':
							repository.autoUpdate = forceBool(value.strip())
						elif option.lower() == 'autosetup':
							repository.autoSetup = forceBool(value.strip())
						elif option.lower() == 'onlydownload':
							repository.onlyDownload = forceBool(value.strip())
						elif option.lower() == 'inheritproductproperties':
							if not opsiDepotId:
								logger.warning(u"InheritProductProperties not possible with normal http ressource.")
								repository.inheritProductProperties = False
							else:
								repository.inheritProductProperties = forceBool(value.strip())
						elif option.lower() == 'dirs':
							repository.dirs = []
							dirs = value.split(',')
							for directory in dirs:
								directory = directory.strip()
								if not directory:
									continue
								repository.dirs.append(forceFilename(directory))
						elif option.lower() == 'excludes':
							repository.excludes = []
							excludes = value.split(',')
							for exclude in excludes:
								exclude = exclude.strip()
								if not exclude:
									continue
								repository.excludes.append(re.compile(exclude))
						elif option.lower() == 'includeproductids':
							repository.includes = []
							includes = value.split(',')
							for include in includes:
								include = include.strip()
								if not include:
									continue
								repository.includes.append(re.compile(include))

					if self.config.get('installAllAvailable'):
						repository.autoInstall = True
						repository.autoUpdate = True
						repository.excludes = []
					self.config['repositories'].append(repository)
				else:
					logger.error(u"Unhandled section '%s'" % section)
		except Exception as exclude:
			raise Exception(u"Failed to read config file '%s': %s" % (self.config["configFile"], exclude))

	def getConfigBackend(self):
		if not self.configBackend:
			self.configBackend = BackendManager(
				dispatchConfigFile=u'/etc/opsi/backendManager/dispatch.conf',
				backendConfigDir=u'/etc/opsi/backends',
				extensionConfigDir=u'/etc/opsi/backendManager/extend.d',
				depotbackend=True,
				hostControlBackend=True
			)
		return self.configBackend

	def getDepotConnection(self, depotId, username, password):
		if depotId not in self.depotConnections:
			self.depotConnections[depotId] = JSONRPCBackend(
				address=depotId,
				username=username,
				password=password
			)
		return self.depotConnections[depotId]

	def processUpdates(self):
		notifier = None
		if self.config["notification"]:
			logger.notice(u"Notification is activated")
			notifier = EmailNotifier(
				smtphost=self.config["smtphost"],
				smtpport=self.config["smtpport"],
				sender=self.config["sender"],
				receivers=self.config["receivers"],
				subject=self.config["subject"],
			)

			if self.config["use_starttls"]:
				notifier.useStarttls = self.config["use_starttls"]

			if self.config["smtpuser"] and self.config["smtppassword"] is not None:
				notifier.username = self.config["smtpuser"]
				notifier.password = self.config["smtppassword"]

		try:
			try:
				if not self.config["repositories"]:
					logger.notice(u"No repositories configured, nothing to do")
					return

				installedProducts = self.getInstalledProducts()
				localPackages = self.getLocalPackages()
				downloadablePackages = self.onlyNewestPackages(self.getDownloadablePackages())
				if self.config["processProductIds"]:
					# Checking if given productIds are available and process only these products
					newProductList = []
					for product in self.config["processProductIds"]:
						found = False
						for pac in downloadablePackages:
							if product == pac["productId"]:
								newProductList.append(pac)
								found = True
								break
						if not found:
							raise Exception(u"You have searched for a product, which was not found in configured repository: '%s'" % product)
					if newProductList:
						downloadablePackages = newProductList
				newPackages = []

				for availablePackage in downloadablePackages:
					logger.info(u"Testing if download/installation of package '%s' is needed" % availablePackage["filename"])
					productInstalled = False
					updateAvailable = False
					installationRequired = False
					for product in installedProducts:
						if product['productId'] == availablePackage['productId']:
							logger.debug(u"Product '%s' is installed" % availablePackage['productId'])
							productInstalled = True
							logger.debug(u"Available product version is '%s', installed product version is '%s-%s'" \
								% (availablePackage['version'], product['productVersion'], product['packageVersion']))
							updateAvailable = compareVersions(availablePackage['version'], '>', '%s-%s' % (product['productVersion'], product['packageVersion']))
							break

					if not productInstalled:
						if availablePackage['repository'].autoInstall:
							logger.notice(u"%s - installation required: product '%s' is not installed and auto install is set for repository '%s'" \
								% (availablePackage["filename"], availablePackage['productId'], availablePackage['repository'].baseUrl))
							installationRequired = True
						else:
							logger.notice(u"%s - installation not required: product '%s' is not installed but auto install is not set for repository '%s'" \
								% (availablePackage["filename"], availablePackage['productId'], availablePackage['repository'].baseUrl))
					elif updateAvailable:
						if availablePackage['repository'].autoUpdate:
							logger.notice(u"%s - installation required: a more recent version of product '%s' was found (installed: %s-%s, available: %s) and auto update is set for repository '%s'" \
									% (availablePackage["filename"], availablePackage['productId'], product['productVersion'], product['packageVersion'], availablePackage['version'], availablePackage['repository'].baseUrl))
							installationRequired = True
						else:
							logger.notice(u"%s - installation not required: a more recent version of product '%s' was found (installed: %s-%s, available: %s) but auto update is not set for repository '%s'" \
									% (availablePackage["filename"], availablePackage['productId'], product['productVersion'], product['packageVersion'], availablePackage['version'], availablePackage['repository'].baseUrl))
					else:
						logger.notice(u"%s - installation not required: installed version '%s-%s' of product '%s' is up to date" \
									% (availablePackage["filename"], product['productVersion'], product['packageVersion'], availablePackage['productId']))

					if not installationRequired:
						continue

					downloadNeeded = True
					localPackageFound = None
					for localPackage in localPackages:
						if localPackage['productId'] == availablePackage['productId']:
							logger.debug(u"Found local package file '%s'" % localPackage['filename'])
							localPackageFound = localPackage
							if localPackage['filename'] == availablePackage['filename']:
								if localPackage['md5sum'] == availablePackage['md5sum']:
									downloadNeeded = False
									break
					if not downloadNeeded:
						logger.notice(u"%s - download of package is not required: found local package %s with matching md5sum" \
									% (availablePackage["filename"], localPackageFound['filename']))
					elif localPackageFound:
						logger.notice(u"%s - download of package is required: found local package %s which differs from available" \
									% (availablePackage["filename"], localPackageFound['filename']))
					else:
						logger.notice(u"%s - download of package is required: local package not found" % availablePackage["filename"])

					packageFile = os.path.join(self.config["packageDir"], availablePackage["filename"])
					zsynced = False
					if downloadNeeded:
						if self.config["zsyncCommand"] and availablePackage['zsyncFile'] and localPackageFound:
							if availablePackage['repository'].baseUrl.split(':')[0].lower().endswith('s'):
								logger.warning(u"Cannot use zsync, because zsync does not support https")
								self.downloadPackage(availablePackage, notifier=notifier)
							else:
								if localPackageFound['filename'] != availablePackage['filename']:
									os.rename(os.path.join(self.config["packageDir"], localPackageFound["filename"]), packageFile)
									localPackageFound["filename"] = availablePackage['filename']
								self.zsyncPackage(availablePackage, notifier=notifier)
								zsynced = True
						else:
							self.downloadPackage(availablePackage, notifier=notifier)
						self.cleanupPackages(availablePackage)

					if availablePackage['md5sum']:
						logger.info(u"Verifying download of package '%s'" % packageFile)
						md5 = md5sum(packageFile)
						if md5 == availablePackage["md5sum"]:
							logger.info(u"Md5sum match, package download verified")
						elif md5 != availablePackage["md5sum"] and zsynced:
							logger.warning(u"zsynced Download has failed, try once to load full package")
							self.downloadPackage(availablePackage, notifier=notifier)
							self.cleanupPackages(availablePackage)

							md5 = md5sum(packageFile)
							if md5 == availablePackage["md5sum"]:
								logger.info(u"Md5sum match, package download verified")
							else:
								raise Exception(u"Failed to download package '%s', md5sum mismatch" % availablePackage['packageFile'])
						else:
							logger.info(u"Md5sum mismatch and no zsync. Doing nothing.")
					else:
						logger.warning(u"Cannot verify download of package: missing md5sum file")

					newPackages.append(availablePackage)

				if not newPackages:
					logger.info(u"No new packages downloaded")
					return

				now = time.localtime()
				now = '%d:%d' % (now[3], now[4])

				def tdiff(t1, t2):
					t1 = int(t1.split(':')[0]) * 60 + int(t1.split(':')[1])
					t2 = int(t2.split(':')[0]) * 60 + int(t2.split(':')[1])
					if t1 > t2:
						return 24 * 60 - t1 + t2

				insideInstallWindow = True
				if not self.config['installationWindowStartTime'] or not self.config['installationWindowEndTime']:
					logger.notice(u"Installation time window not defined, installing products and setting actions")
				elif tdiff(self.config['installationWindowStartTime'], self.config['installationWindowEndTime']) >= tdiff(self.config['installationWindowStartTime'], now):
					logger.notice(u"We are inside the installation time window, installing products and setting actions")
				else:
					logger.notice(u"We are outside installation time window, not installing products except for product ids %s" \
							% self.config['installationWindowExceptions'])
					insideInstallWindow = False

				sequence = []
				for package in newPackages:
					if not insideInstallWindow and not package['productId'] in self.config['installationWindowExceptions']:
						continue
					sequence.append(package['productId'])

				for package in newPackages:
					if not package['productId'] in sequence:
						continue
					packageFile = os.path.join(self.config["packageDir"], package["filename"])
					productId = package['productId']
					ppf = ProductPackageFile(packageFile, tempDir=self.config.get('tempdir', '/tmp'))
					ppf.getMetaData()
					dependencies = ppf.packageControlFile.getPackageDependencies()
					ppf.cleanup()
					for dependency in dependencies:
						try:
							ppos = sequence.index(productId)
							dpos = sequence.index(dependency['package'])
							if ppos < dpos:
								sequence.remove(dependency['package'])
								sequence.insert(ppos, dependency['package'])
						except Exception as error:
							logger.debug(u"While processing package '%s', dependency '%s': %s" % (packageFile, dependency['package'], error))

				sortedPackages = []
				for productId in sequence:
					for package in newPackages:
						if productId == package['productId']:
							sortedPackages.append(package)
							break
				newPackages = sortedPackages

				installedPackages = []
				for package in newPackages:
					packageFile = os.path.join(self.config["packageDir"], package["filename"])

					if package['repository'].onlyDownload:
						continue

					propertyDefaultValues = {}
					try:
						if package['repository'].inheritProductProperties and availablePackage['repository'].opsiDepotId:
							logger.info(u"Trying to get product property defaults from repository")
							productPropertyStates = self.getConfigBackend().productPropertyState_getObjects(
											productId=package['productId'],
											objectId=availablePackage['repository'].opsiDepotId)
						else:
							productPropertyStates = self.getConfigBackend().productPropertyState_getObjects(
											productId=package['productId'],
											objectId=self.depotId)
						if productPropertyStates:
							for pps in productPropertyStates:
								propertyDefaultValues[pps.propertyId] = pps.values
						logger.notice(u"Using product property defaults: %s" % propertyDefaultValues)
					except Exception as error:
						logger.warning(u"Failed to get product property defaults: %s" % error)

					logger.notice(u"Installing package '%s'" % packageFile)
					self.getConfigBackend().depot_installPackage(filename=packageFile, force=True, propertyDefaultValues=propertyDefaultValues, tempDir=self.config.get('tempdir', '/tmp'))
					productOnDepots = self.getConfigBackend().productOnDepot_getObjects(depotId=self.depotId, productId=package['productId'])
					if not productOnDepots:
						raise Exception(u"Product '%s' not found on depot '%s' after installation" % (package['productId'], self.depotId))
					package['product'] = self.getConfigBackend().product_getObjects(
						id=productOnDepots[0].productId,
						productVersion=productOnDepots[0].productVersion,
						packageVersion=productOnDepots[0].packageVersion
					)[0]

					if notifier:
						notifier.appendLine(u"Package '%s' successfully installed" % packageFile)
					logger.notice(u"Package '%s' successfully installed" % packageFile)
					installedPackages.append(package)

				if not installedPackages:
					logger.info(u"No new packages installed")
					return

				wakeOnLanClients = []
				shutdownProduct = None
				if self.config['wolAction'] and self.config["wolShutdownWanted"]:
					for product in self.getConfigBackend().productOnDepot_getObjects(depotId=self.depotId, productId='shutdownwanted'):
						shutdownProduct = product
						logger.info(u"Found 'shutdownwanted' product on depot '%s': %s" % (self.depotId, shutdownProduct))
						break
					if not shutdownProduct:
						logger.error(u"Product 'shutdownwanted' not avaliable on depot '%s'" % self.depotId)

				for package in installedPackages:
					if not package['product'].setupScript:
						continue
					if package['repository'].autoSetup:
						if isinstance(package['product'], NetbootProduct):
							logger.notice(u"Not setting action 'setup' for product '%s' where installation status 'installed' because auto setup is not allowed for netboot products" \
									% (package['productId']))
							continue

						logger.notice(u"Setting action 'setup' for product '%s' where installation status 'installed' because auto setup is set for repository '%s'" \
							% (package['productId'], package['repository'].baseUrl))
					else:
						logger.notice(u"Not setting action 'setup' for product '%s' where installation status 'installed' because auto setup is not set for repository '%s'" \
							% (package['productId'], package['repository'].baseUrl))
						continue

					clientToDepotserver = self.getConfigBackend().configState_getClientToDepotserver(depotIds=[self.depotId])
					if clientToDepotserver:
						clientIds = []
						for ctd in clientToDepotserver:
							if ctd['clientId'] and not ctd['clientId'] in clientIds:
								clientIds.append(ctd['clientId'])
						if clientIds:
							productOnClients = self.getConfigBackend().productOnClient_getObjects(
								attributes=['installationStatus'],
								productId=[package['productId']],
								productType=['LocalbootProduct'],
								clientId=clientIds,
								installationStatus=['installed'],
							)
							if productOnClients:
								for i in range(len(productOnClients)):
									productOnClients[i].setActionRequest('setup')
									if self.config['wolAction'] and not package['productId'] in self.config['wolActionExcludeProductIds']:
										if not productOnClients[i].clientId in wakeOnLanClients:
											wakeOnLanClients.append(productOnClients[i].clientId)
								self.getConfigBackend().productOnClient_updateObjects(productOnClients)

				if wakeOnLanClients:
					logger.notice(u"Powering on clients %s" % wakeOnLanClients)
					for clientId in wakeOnLanClients:
						try:
							logger.info(u"Powering on client '%s'" % clientId)
							if self.config["wolShutdownWanted"] and shutdownProduct:
								logger.info(u"Setting shutdownwanted to 'setup' for client '%s'" % clientId)

								self.getConfigBackend().productOnClient_updateObjects(
									ProductOnClient(
										productId=shutdownProduct.productId,
										productType=shutdownProduct.productType,
										productVersion=shutdownProduct.productVersion,
										packageVersion=shutdownProduct.packageVersion,
										clientId=clientId,
										actionRequest='setup'
									)
								)
							self.getConfigBackend().hostControl_start(hostIds=[clientId])
							time.sleep(self.config["wolStartGap"])
						except Exception as error:
							logger.error(u"Failed to power on client '%s': %s" % (clientId, error))
			except Exception as error:
				if notifier:
					notifier.appendLine(u"Error occurred: %s" % error)
				raise
		finally:
			if notifier and notifier.hasMessage():
				notifier.notify()

	def zsyncPackage(self, availablePackage, notifier=None):
		outFile = os.path.join(self.config["packageDir"], availablePackage["filename"])
		curdir = os.getcwd()
		os.chdir(os.path.dirname(outFile))
		try:
			logger.notice(u"Zsyncing %s to %s" % (availablePackage["packageFile"], outFile))

			cmd = u"%s -A %s='%s:%s' -o '%s' %s 2>&1" % (
				self.config["zsyncCommand"],
				availablePackage['repository'].baseUrl.split('/')[2].split(':')[0],
				availablePackage['repository'].username,
				availablePackage['repository'].password,
				outFile,
				availablePackage["zsyncFile"]
			)

			if availablePackage['repository'].proxy:
				cmd = u"http_proxy=%s %s" % (availablePackage['repository'].proxy, cmd)

			stateRegex = re.compile('\s([\d\.]+)%\s+([\d\.]+)\skBps(.*)$')
			data = ''
			percent = 0.0
			speed = 0
			handle = System.execute(cmd, getHandle=True)
			while True:
				inp = handle.read(16)
				if not inp:
					handle.close()
					break
				data += inp
				match = stateRegex.search(data)
				if not match:
					continue
				data = match.group(3)
				if (percent == 0) and (float(match.group(1)) == 100):
					continue
				percent = float(match.group(1))
				speed = float(match.group(2)) * 8
				logger.debug(u'Zsyncing %s: %d%% (%d kbit/s)' % (availablePackage["packageFile"], percent, speed))
			if notifier:
				notifier.appendLine(u"Zsync of '%s' completed" % availablePackage["packageFile"])
			logger.notice(u"Zsync of '%s' completed" % availablePackage["packageFile"])
		finally:
			os.chdir(curdir)

	def downloadPackage(self, availablePackage, notifier=None):
		url = availablePackage["packageFile"]
		outFile = os.path.join(self.config["packageDir"], availablePackage["filename"])

		passwordManager = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passwordManager.add_password(None, availablePackage['repository'].baseUrl, availablePackage['repository'].username, availablePackage['repository'].password)
		handler = urllib2.HTTPBasicAuthHandler(passwordManager)
		if availablePackage['repository'].proxy:
			logger.notice(u"Using Proxy: %s" % availablePackage['repository'].proxy)
			proxy_handler = urllib2.ProxyHandler({'http': availablePackage['repository'].proxy, 'https': availablePackage['repository'].proxy})
			opener = urllib2.build_opener(proxy_handler, handler)
		else:
			opener = urllib2.build_opener(handler)
		urllib2.install_opener(opener)

		req = urllib2.Request(url, None, self.httpHeaders)
		con = opener.open(req)
		size = int(con.info().get('Content-length', 0))
		if size:
			logger.notice(u"Downloading %s (%s MB) to %s" % (url, round(size / (1024.0 * 1024.0), 2), outFile))
		else:
			logger.notice(u"Downloading %s to %s" % (url, outFile))

		completed = 0.0
		percent = 0.0
		lastTime = time.time()
		lastCompleted = 0
		lastPercent = 0
		speed = 0

		with open(outFile, 'wb') as out:
			while True:
				chunk = con.read(32768)
				if not chunk:
					break
				completed += len(chunk)
				out.write(chunk)

				if size > 0:
					try:
						percent = round(100 * completed / size, 1)
						if lastPercent != percent:
							lastPercent = percent
							now = time.time()
							if not speed or (now - lastTime) > 2:
								speed = 8 * int(((completed - lastCompleted) / (now - lastTime)) / 1024)
								lastTime = now
								lastCompleted = completed
							logger.debug(u'Downloading {0}: {1:d}% ({2:d} kbit/s)'.format(url, percent, speed))
					except Exception:
						pass

		if notifier:
			if size:
				notifier.appendLine(u"Download of '%s' completed (~ %s MB)" % (url, round(size / (1024.0 * 1024.0), 2)))
			else:
				notifier.appendLine(u"Download of '%s' completed" % url)
		logger.notice(u"Download of '%s' completed" % url)

	def cleanupPackages(self, newPackage):
		logger.info(u"Cleaning up in %s" % self.config["packageDir"])

		try:
			setRights(self.config["packageDir"])
		except Exception as error:
			logger.warning(u"Failed to set rights on directory '{0}': {1}".format(self.config["packageDir"], error))

		for f in os.listdir(self.config["packageDir"]):
			path = os.path.join(self.config["packageDir"], f)
			if not os.path.isfile(path):
				continue
			if path.endswith('.zs-old'):
				os.unlink(path)
				continue

			try:
				productId = '_'.join(f.split('_')[:-1])
				version = f[:-5].split('_')[-1]
			except Exception:
				continue

			if productId == newPackage["productId"] and version != newPackage["version"]:
				logger.info(u"Deleting obsolete package file '%s'" % path)
				os.unlink(path)

		packageFile = os.path.join(self.config["packageDir"], newPackage["filename"])

		md5sumFile = u'{package}.md5'.format(package=packageFile)
		logger.info(u"Creating md5sum file '%s'" % md5sumFile)

		with open(md5sumFile, 'w') as f:
			f.write(md5sum(packageFile))

		setRights(md5sumFile)

		try:
			zsyncFile = u'{package}.zsync'.format(package=packageFile)
			logger.info(u"Creating zsync file '%s'" % zsyncFile)
			zsyncFile = ZsyncFile(zsyncFile)
			zsyncFile.generate(packageFile)
		except Exception as error:
			logger.error(u"Failed to create zsync file '%s': %s" % (zsyncFile, error))

	def onlyNewestPackages(self, packages):
		newestPackages = []
		for package in packages:
			found = None
			for i in range(len(newestPackages)):
				if newestPackages[i]['productId'] == package['productId']:
					found = i
					break
			if found is None:
				newestPackages.append(package)
			elif compareVersions(package['version'], '>', newestPackages[i]['version']):
				logger.debug("Package version '%s' is newer than version '%s'" % (package['version'], newestPackages[i]['version']))
				newestPackages[i] = package
		return newestPackages

	def getLocalPackages(self):
		logger.notice(u"Getting info for local packages in '%s'" % self.config["packageDir"])
		packages = []
		for f in os.listdir(self.config["packageDir"]):
			if not f.endswith('.opsi'):
				continue
			packageFile = os.path.join(self.config["packageDir"], f)
			logger.info(u"Found local package '%s'" % packageFile)
			try:
				productId = '_'.join(f.split('_')[:-1])
				version = f[:-5].split('_')[-1]
				packages.append(
					{
						"productId": productId.lower(),
						"version": version,
						"packageFile": packageFile,
						"filename": f,
						"md5sum": md5sum(packageFile)
					}
				)
				logger.debug(u"Local package info: %s" % packages[-1])
			except Exception as e:
				logger.error("Failed to process file '%s': %s" % (f, e))
		return packages

	def getInstalledProducts(self):
		logger.notice(u"Getting installed products")
		products = []
		configBackend = self.getConfigBackend()
		for productOnDepot in configBackend.productOnDepot_getObjects(depotId=self.depotId):
			product = productOnDepot.toHash()
			logger.info(u"Found installed product '%s_%s-%s'" % (product['productId'], product['productVersion'], product['packageVersion']))
			products.append(product)
		return products

	def getDownloadablePackages(self):
		downloadablePackages = []
		for repository in self.config.get("repositories", []):
			downloadablePackages.extend(self.getDownloadablePackagesFromRepository(repository))
		return downloadablePackages

	def getDownloadablePackagesFromRepository(self, repository):
		logger.notice(u"Getting package infos from repository '%s'" % repository.baseUrl)
		packages = []

		depotConnection = None
		depotRepositoryPath = None
		if repository.opsiDepotId:
			depotConnection = self.getDepotConnection(repository.opsiDepotId, repository.username, repository.password)
			repositoryLocalUrl = depotConnection.getDepot_hash(repository.opsiDepotId).get("repositoryLocalUrl")
			logger.info(u"Got repository local url '%s' for depot '%s'" % (repositoryLocalUrl, repository.opsiDepotId))
			if not repositoryLocalUrl or not repositoryLocalUrl.startswith('file://'):
				raise Exception(u"Invalid repository local url for depot '%s'" % repository.opsiDepotId)
			depotRepositoryPath = repositoryLocalUrl[7:]

		passwordManager = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passwordManager.add_password(None, repository.baseUrl.encode('utf-8'), repository.username.encode('utf-8'), repository.password.encode('utf-8'))
		handler = urllib2.HTTPBasicAuthHandler(passwordManager)
		if repository.proxy:
			logger.notice(u"Using Proxy: %s" % repository.proxy)
			proxy_handler = urllib2.ProxyHandler(
				{
					'http': repository.proxy,
					'https': repository.proxy
				}
			)
			opener = urllib2.build_opener(proxy_handler, handler)
		else:
			opener = urllib2.build_opener(handler)
		urllib2.install_opener(opener)

		for url in repository.getDownloadUrls():
			try:
				url = urllib.quote(url.encode('utf-8'), safe="/#%[]=:;$&()+,!?*@'~")
				req = urllib2.Request(url, None, self.httpHeaders)
				response = opener.open(req)
				content = response.read()
				logger.debug("content: '%s'" % content)
				format = formatter.NullFormatter()
				htmlParser = LinksExtractor(format)
				htmlParser.feed(content)
				htmlParser.close()
				for link in htmlParser.getLinks():
					if not link.endswith('.opsi'):
						continue

					excluded = False
					included = True

					if repository.includes:
						included = False
						for include in repository.includes:
							if include.search(link):
								included = True
								break
					if not included:
						logger.info(u"Package '%s' is not included. Please check your includeProductIds-entry in configurationfile." % link)
						continue

					for exclude in repository.excludes:
						if exclude.search(link):
							excluded = True
							break
					if excluded:
						logger.info(u"Package '%s' excluded by regular expression" % link)
						continue
					try:
						productId = '_'.join(link.split('_')[:-1])
						version = link[:-5].split('_')[-1]
						packageFile = url + '/' + link
						logger.info(u"Found opsi package: %s" % packageFile)
						packages.append(
							{
								"repository": repository,
								"productId": productId.lower(),
								"version": version,
								"packageFile": packageFile,
								"filename": link,
								"md5sum": None,
								"zsyncFile": None
							}
						)
						if depotConnection:
							packages[-1]["md5sum"] = depotConnection.getMD5Sum(u'%s/%s' % (depotRepositoryPath, link))
						logger.debug(u"Repository package info: %s" % packages[-1])
					except Exception as error:
						logger.error(u"Failed to process link '%s': %s" % (link, error))

				if not depotConnection:
					for link in htmlParser.getLinks():
						isMd5 = link.endswith('.opsi.md5')
						isZsync = link.endswith('.opsi.zsync')
						try:
							filename = None
							if isMd5:
								filename = link[:-4]
							elif isZsync:
								filename = link[:-6]
							else:
								continue
							for i in range(len(packages)):
								if packages[i].get('filename') == filename:
									if isMd5:
										req = urllib2.Request(url + '/' + link, None, self.httpHeaders)
										con = opener.open(req)
										md5sum = con.read(32768)
										match = re.search('([a-z\d]{32})', md5sum)
										if match:
											packages[i]["md5sum"] = match.group(1)
											logger.debug(u"Got md5sum for package '%s': %s" % (filename, packages[i]["md5sum"]))
									elif isZsync:
										packages[i]["zsyncFile"] = url + '/' + link
										logger.debug(u"Found zsync file for package '%s': %s" % (filename, packages[i]["zsyncFile"]))
									break
						except Exception as error:
							logger.error(u"Failed to process link '%s': %s" % (link, error))
			except Exception as error:
				logger.logException(error, LOG_INFO)
				raise Exception(u"Failed to process url '%s': %s" % (url, error))
		return packages


def usage():
	print(u"")
	print(u"Usage: %s [options]" % os.path.basename(sys.argv[0]))
	print(u"Options:")
	print(u"    -h    Show this help text")
	print(u"    -v    Increase verbosity (can be used multiple times)")
	print(u"    -V    Show version information and exit")
	print(u"    -c    Location of config file")
	print(u"    -i    Install all downloadable packages from configured repositories (ignores excludes)")
	print(u"    -p    List of productIds that will be processed: opsi-winst,opsi-client-agent")
	print(u"")


def main(argv):
	consoleLevel = LOG_ERROR
	logger.setConsoleLevel(consoleLevel)

	config = {
		"userAgent": 'opsi product updater %s' % __version__,
		"packageDir": '/var/lib/opsi/products',
		"configFile": '/etc/opsi/opsi-product-updater.conf',
		"notification": False,
		"smtphost": u'localhost',
		"smtpport": 25,
		"smtpuser": None,
		"smtppassword": None,
		"subject": u'opsi-product-updater',
		"use_starttls": False,
		"sender": u'opsi@localhost',
		"receivers": [],
		"wolAction": False,
		"wolActionExcludeProductIds": [],
		"wolShutdownWanted": False,
		"wolStartGap": 0,
		"installationWindowStartTime": None,
		"installationWindowEndTime": None,
		"installationWindowExceptions": None,
		"repositories": [],
		"installAllAvailable": False,
		"zsyncCommand": None,
		"processProductIds": None
	}
	# Get options
	try:
		(opts, args) = getopt.getopt(argv, "hivVc:p:")
	except getopt.GetoptError:
		usage()
		sys.exit(1)

	for (opt, arg) in opts:
		if opt == "-h":
			usage()
			return
		elif opt == "-V":
			print(__version__)
			return
		elif opt == "-c":
			config["configFile"] = arg
		elif opt == "-v":
			consoleLevel += 1
			logger.setConsoleLevel(consoleLevel)
		elif opt == "-i":
			config["installAllAvailable"] = True
		elif opt == "-p":
			config["processProductIds"] = arg.split(",")
		else:
			raise Exception(u"Unknown parameter given: '%s'" % opt)

	try:
		config["zsyncCommand"] = System.which("zsync")
		logger.notice(u"Zsync command found: %s" % config["zsyncCommand"])
	except Exception:
		logger.info(u"Zsync command not found")

	pid = os.getpid()
	running = None
	try:
		for anotherPid in System.execute("%s -x %s" % (System.which("pidof"), os.path.basename(sys.argv[0])))[0].strip().split():
			if int(anotherPid) != pid:
				running = anotherPid
	except Exception as error:
		logger.error(error)
	if running:
		raise Exception(u"Another %s process is running (pid: %s)." % (os.path.basename(sys.argv[0]), running))

	opu = OpsiPackageUpdater(config)
	opu.processUpdates()


if __name__ == "__main__":
	logger.setConsoleColor(True)

	try:
		main(sys.argv[1:])
	except SystemExit:
		pass
	except KeyboardInterrupt:
		sys.exit(1)
	except Exception as e:
		logger.logException(e)
		print(u"ERROR: {0}".format(forceUnicode(e).encode('utf-8')), file=sys.stderr)
		sys.exit(1)

	sys.exit(0)
