fixed encoding error in milter

scripts/sortmilter

@@ -134,7 +134,7 @@ class Letter:
         else:
             Logger().printDEBUG(
                 "Wrong file or content or message: '%s', '%s', '%s'",
-                    str(file),str(content),str(message))
+                    str(file), str(content), str(message))
 
     def _getAddressesFrom(self,field):
         return getaddresses([self.mail.get(field,"")])
@@ -182,21 +182,27 @@ class Letter:
 
     def _getTextPlain(self):
         """Get content from body message"""
-        letter_text = self.mail.get_payload(decode=True)
-        charset = self.mail.get_charsets()
-        # if charset specified and it is not utf-8
-        try:
-            if charset and charset[0] and not charset[0].lower() in ['utf-8', 'utf8']:
-                letter_text = letter_text.decode(charset[0]).encode('utf-8')
-        except Exception as e:
-            Logger().printDEBUG("wrong decode from %s: %s" % (charset[0], e))
-        return letter_text
+        # letter_text = self.mail.get_payload(decode=True)
+        # charset = self.mail.get_charsets()
+        # # if charset specified and it is not utf-8
+        # try:
+        #     if charset and charset[0] and not charset[0].lower() in ['utf-8', 'utf8']:
+        #         letter_text = letter_text.decode(charset[0])
+        # except Exception as e:
+        #     Logger().printDEBUG("wrong decode from %s: %s" % (charset[0], e))
+
+        letter_text = self.mail.get_payload()
+        if isinstance(letter_text, str):
+            return letter_text
+        else:
+            Logger().printDEBUG("wrong content type (expected plain text)")
+
 
     def getSenderEmails(self):
         """Cache function for _getSenderEmails"""
         if not self.cacheSender:
             self.cacheSender = self._getSenderEmails()
-        Logger().printDEBUG("Sender:%s"%self.cacheSender)
+        Logger().printDEBUG("Sender:%s" % self.cacheSender)
         return self.cacheSender
 
     def _getSenderEmails(self):
@@ -249,17 +255,15 @@ class Letter:
     def getSubject(self):
         """Get 'Subject' field"""
         try:
-            decodedSubject =email.header.decode_header(self.mail.get('Subject',
+            decodedSubject = email.header.decode_header(self.mail.get('Subject',
                 self.defaults['Subject']))
             # join to string
             return "".join(
-                [
                     # decode first element by second element
-                    i[0].decode(i[1] or "utf-8").encode('utf-8')
-                    # decoded header
+                    i[0].decode(i[1] or "utf-8") if i[1] else i[0]
                     for i in decodedSubject
-                ])
-        except:
+                )
+        except Exception as e:
             Logger().printDEBUG("Wrong subject:%s" % self.mail.get("Subject"))
             return self.defaults['Subject']
 
@@ -276,22 +280,22 @@ class Letter:
         maxfilename = 100
         filename = ("%s_%s%s" % \
             (time.strftime("%Y-%m-%d_%H:%M", time.localtime(self.getDate())),
-                        suffix, self.getSubject())).replace('/','')
-        return filename.decode('utf-8')[:maxfilename].encode('utf-8')
+                        suffix, self.getSubject())).replace('/', '')
+        return filename[:maxfilename]
 
-    def copyTo(self,destination,suffix="",filename=None):
-        destination = pathjoin(destination,filename or self._genfilename(suffix))
+    def copyTo(self, destination, suffix="", filename=None):
+        destination = pathjoin(destination, filename or self._genfilename(suffix))
         # if destination file exists, then rename by addition number
         if pathexists(destination):
             i = 2
-            while pathexists("%s%02d"%(destination,i)):
+            while pathexists("%s%02d" % (destination, i)):
                 i += 1
-            destination = "%s%02d"%(destination,i)
-        f = open(destination,"w")
+            destination = "%s%02d" % (destination, i)
+        f = open(destination, "w")
         f.write(self.mail.as_string())
         f.close()
         # set mtime to letter file by letter date
-        os.utime(destination, (self.getDate(),)*2)
+        os.utime(destination, (self.getDate(),) * 2)
 
 class MailOpponent:
     """Mail opponent"""
@@ -329,7 +333,7 @@ class MailOpponent:
         else:
             letter.copyTo(self.getDirectory(),suffix="In:")
         self.date = max(self.date, letter.getDate())
-        os.utime(self.getDirectory(),(self.date,)*2)
+        os.utime(self.getDirectory(), (self.date,) * 2)
 
 class MailBox:
     """Mail box. Keep mail sorted by opponents. (opponent union)"""
@@ -630,7 +634,7 @@ def main(argv):
     if parser.values.log == "-":
         Logger().logfile = sys.stdout
     else:
-        Logger().logfile = open(parser.values.log,"a")
+        Logger().logfile = open(parser.values.log, "a")
 
     # create mail keeper by storage_dir parameter
     mail_keeper = MailKeeper(storagedir=parser.values.storage_dir,
@@ -649,18 +653,21 @@ def main(argv):
         # process all letter specified by letter-file option
         for lettername in parser.values.letter_file:
             if pathexists(lettername):
-                mail_keeper.addLetter( Letter(file=lettername) )
+                ll = Letter(file=lettername)
+                mail_keeper.addLetter(ll)
+                #DEBUG
+                print(ll.mail.get_content_type())
             else:
-                Logger().printERROR("file not found:%s"%lettername)
+                Logger().printERROR("file not found:%s" % lettername)
 
         # process all mail directories specified by letters-dir option
         for directory in parser.values.letters_dir:
             for letter in getMailFromFolder(directory):
                 try:
-                    Logger().printDEBUG("processed file:%s"%letter.getFile())
+                    Logger().printDEBUG("processed file:%s" % letter.getFile())
                     mail_keeper.addLetter(letter)
                     if parser.values.remove_success:
-                        Logger().printDEBUG("removed file:%s"%letter.getFile())
+                        Logger().printDEBUG("removed file:%s" % letter.getFile())
                         os.remove(letter.getFile())
                 except:
                     dbg_except()       

setup.py

@@ -69,7 +69,6 @@ def scanDirs(profilesDirs):
 
 def create_data_files (data_dirs, prefix=""):
     test1_files = []
-    data_files = []
     dirs = scanDirs(data_dirs)
     i = 0
     for obj in dirs: