textFct.py

#!/usr/bin/python3.6
# -*- coding: utf-8 -*-

letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZaàâbcdeéêèëfghiîïjkmlmnoôpqrstuûvwxyz0123456789-\xe7\xc7'
uppercaseLetters = ('aA', 'àA', 'bB', 'cC', '\xe7\xc7', 'dD', 'eE', 'éE', 'èE', 'êE', 'ëE', 'fF', 'gG', 'hH', 'iI', 'îI', 'ïI', 'jJ', 'kK', 'lL', 'mM', 'nN', 'oO', '\xf4\xe4', 'pP', 'qQ', 'rR', 'sS', 'tT', 'uU', 'vV', 'wW', 'xX', 'yY', 'zZ')

# liste des points, des chaines de caracteres suivies par une majuscule
wordsBeginMaj =( 'paris', 'rueil', 'avon', 'valo', 'leto', 'mars', 'mai', 'juin', 'papa', 'papi', 'victo', 'france', 'lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi', 'dimanche', 'janvier', 'février', 'avril', 'juillet', 'août', 'aout', 'septembre', 'octobre', 'novembre', 'décembre', 'decembre', 'deborah', 'powers', 'cadiot', 'maman', 'mamie', 'régine', 'tony', 'robert', 'simplon', 'loïc', 'jared', 'ville valo', 'shelby', 'magritte', 'gabin', 'makaron', 'malmaison', 'fontainebleau', 'issy', 'moulineaux', 'châte', 'chateaudun', 'michelet', 'chatelet', 'c:\\', 'users\\', 'desktop\\')
wordsBeginMin =( 'Deborah.powers', 'Deborah.noisetier', 'Http', 'File:///', '\nPg_')
wordsFullMaj =( '\\lenovo\\', 'i3f' )
codeKeywords =(
	'set schema', 'declare', 'begin', 'do $$', 'update', 'select', 'from', 'inner join', 'outer join', 'left outer join', 'where', 'and', 'with', 'union', 'when', 'case',
	'having', 'group by', 'order by', 'insert into', 'if', 'elseif', 'end', 'loop', 'perform', 'drop',
	'cd', 'psql', 'git', 'return', 'mvn', 'python', 'else',
	'def', 'class', 'console.log', 'var', 'function', 'private', 'protected', 'public',
	'log.debug', 'log.info'
)
weirdChars =(
	('« ', '"'), (' »', '"'), ('«', '"'), ('»', '"'), ('–', '-'), ('‘', "'"), ('’', "'"), ('“', '"'), ('”', '"'), ('"', '"'), ('…', '...'), ('…', '...'), ('…', '...'),
	('\n ', '\n'), ('\r', ''), (' \n', '\n'), ("\\'", "'"), ('\\n', '\n'), ('\\r', ''), ('\\t', '\t'),
	('\\u00c2', 'Â'), ('\\u00ca', 'Ê'), ('\\u00cb', 'Ë'), ('\\u00ce', 'Î'), ('\\u00cf', 'Ï'), ('\\u00d4', 'Ô'), ('\\u00d6', 'Ö'), ('\\u00db', 'Û'), ('\\u00e0', 'à'), ('\\u00e2', 'â'), ('\\u00e7', 'ç'), ('\\u00e8', 'è'), ('\\u00e9', 'é'), ('\\u00ea', 'ê'), ('\\u00eb', 'ë'), ('\\u00ee', 'î'), ('\\u00ef', 'ï'), ('\\u00f4', 'ô'), ('\\u00f6', 'ö'), ('\\u00fb', 'û'),
	('\x85', '.'), ('\x92', "'"), ('\x96', '"'), ('\x97', "'"), ('\x9c', ' '), ('\xa0', ' '),
	('à', 'à'), ('â', 'â'), ('ç', 'ç'), ('é', 'é'), ('è', 'è'), ('ê', 'ê'), ('î', 'î'), ('ï', 'ï'), ('ô', 'ô'), ('ù', 'ù'), ('û', 'û'), (''', "'"),
	('&mdash;', ' '), ('&nbsp;', ''), ('&oelig;', 'oe'), ('&quot;', ''), ('&lt;', '<'), ('&gt;', '>'), ('&lsquo;', '"'), ('&ldquo;', '"'), ('&rdquo;', '"'), ('&rsquo;', "'"), ('&laquo;', '"'), ('&raquo;', '"'), ('&#8220;', '"'), ('&#8221;', '"'), ('&#8211;', '-'),
	('&amp;', '&'), ('&#x27;', "'"), ('&#039', "'"), ('&#160;', ' '), ('&#xa0;', " "), ('&#39;', "'"), ('&#8217;', "'"), ('\n" ', '\n"'),
	('<br>', '<br/>'), ('<hr>', '<hr/>'), ('File:///', ""), ('file:///', ""), ('c:\\', 'C:\\'), ('C:/', 'C:\\'), ('c:/', 'C:\\')
)
urlWords =(('. gif', '.gif'), ('. com', '.com'), ('. org', '.org'), ('. net', '.net'), ('. fr', '.fr'), ('. ico', '.ico'),
	(': /', ':/'), (': \\', ':\\'), ('localhost: ', 'localhost:'), (': 80', ':80'), ('www. ', 'www.'),
	('. bmp', '.bmp'), ('. jpeg', '.jpeg'), ('. jpg', '.jpg'), ('. png', '.png'), ('. css', '.css'), ('. js', '.js')
)
titleChars = '=*-_#+~'

# ________________________ ma mise en forme perso ________________________

def findEndUrl (text, pos=0):
#	charEndUrl = '\n\t \'",;!()[]{}'
	charEndUrl = '\n\t \'",;!(){}'
	lenText = len (text) +1
	posEnd = lenText
	posTmp = lenText
	for char in charEndUrl:
		posTmp = text.find (char, pos)
		if posTmp >0 and posTmp < posEnd: posEnd = posTmp
	return posEnd

def upperCaseIntern (text):
	text ='\n'+ text
	points =( '\n', '. ', '! ', '? ', ': ', ':\t', '(', '{', '[',
		'\n_ ', '\n* ', '\n- ', '\n--> ', '\n\t', '++ ' '## ', '__ ', '-- ', '** ', '== ')
	for i, j in uppercaseLetters:
		for p in points: text = text.replace (p+i, p+j)
	for word in wordsBeginMin: text = text.replace (word, word.lower())
	for word in wordsBeginMaj[12:]: text = text.replace (word, word.capitalize())
	for word in wordsFullMaj: text = text.replace (word, word.upper())
	text = text.replace ('', '')
	pointsEnd ='\n\t /\\:.,;!?}])'
	pointsStart ='\n\t ([{'
	for word in wordsBeginMaj[:12]:
		for s in pointsStart:
			for e in pointsEnd: text = text.replace (s+ word +e, s+ word.capitalize() +e)
	text = text.replace ('\\lenovo\\', '\\LENOVO\\')
	# le code
	for artefact in codeKeywords:
		text = text.replace ('\n'+ artefact.capitalize() +' ', '\n'+ artefact +' ')
		text = text.replace ('\t'+ artefact.capitalize() +' ', '\t'+ artefact +' ')
		text = text.replace ('\n'+ artefact.capitalize() +'(', '\n'+ artefact +' (')
		text = text.replace ('\t'+ artefact.capitalize() +'(', '\t'+ artefact +' (')
		text = text.replace ('\n'+ artefact.capitalize() +'\n', '\n'+ artefact +'\n')
		text = text.replace ('\t'+ artefact.capitalize() +'\n', '\t'+ artefact +'\n')
	text = text.strip()
	while "  " in text: text = text.replace ("  "," ")
	return text

def upperCase (text, case=""):
	"""	rest: je supprime l'ancienne casse
		upper: je rajoute les majuscules
	"""
	# préparer le texte
	text = '\n'+ text +'\n'
	text = text.replace ('\ncode\n', '\ncode\n')
	if '\ncode\t' in text:
		paragraphList = text.split ('\ncode\t')
		paragraphRange = range (1, len (paragraphList))
		for i in paragraphRange: paragraphList[i] = paragraphList[i].replace ('\n', '\n/\n', 1)
		text = '\ncode\n'.join (paragraphList)
	# isoler les blocs devant rester intacts
	if '\ncode\n' in text:
		paragraphList = text.split ('\ncode\n')
		if 'reset' in case: paragraphList[0] = paragraphList[0].lower()
		if 'upper' in case: paragraphList[0] = upperCaseIntern (paragraphList[0])
		paragraphRange = range (1, len (paragraphList))
		for i in paragraphRange:
			d= paragraphList[i].find ('\n/\n') +1
			temp = paragraphList[i][d:]
			paragraphList[i] = paragraphList[i][:d]
			if 'reset' in case: temp = temp.lower()
			if 'upper' in case: temp = upperCaseIntern (temp)
			paragraphList[i] = paragraphList[i] + temp
		text = '\ncode\n'.join (paragraphList)
	else:
		if 'reset' in case: text = text.lower()
		if 'upper' in case: text = upperCaseIntern (text)
	text = text.strip()
	return text

def simpleSpace (text):
	while "  " in text: text = text.replace ("  ", " ")
	return text

def cleanBasic (text):
	for i, j in weirdChars: text = text.replace (i, j)
	text = text.strip()
	while "  " in text: text = text.replace ("  ", " ")
	text = text.replace ('\n ', '\n')
	text = text.replace (' \n', '\n')
	text = text.replace ('\t ', '\t')
	text = text.replace (' \t', '\t')
	while '\t\n' in text: text = text.replace ('\t\n', '\n')
	while '\n\n' in text: text = text.replace ('\n\n', '\n')
	return text

def cleanHtml (text):
	text = cleanBasic (text)
	text = text.replace ('\n', ' ')
	text = text.replace ('\t', ' ')
	text = text.strip()
	while "  " in text: text = text.replace ("  ", " ")
	text = text.replace ('> ', '>')
	text = text.replace (' <', '<')
	text = text.replace (' >', '>')
	text = text.replace (' />', '/>')
	# supprimer les br en trop
	while '<br/><br/>' in text: text = text.replace ('<br/><br/>', '<br/>')
	text = text.replace ('<br/><', '<')
	text = text.replace ('><br/>', '>')
	if '</table>' not in text and '</li>' not in text: text = text.replace ('<br/>', '</p><p>')
	# supprimer les icones
	icons = '🔸🔹➡️ℹ🌐📦🔒⚠️🚧👆📂📄📨✅'
	for icon in icons[:2]: text = text.replace (icon, ", ")
	for icon in icons[2:]: text = text.replace (icon, " ")
	while "  " in text: text = text.replace ("  "," ")
	# nettoyage plus avancé
	points = '.,)'
	for p in points: text = text.replace (" "+p, p)
	text = text.replace ("( ", '(')
	return text

def cleanCss (text):
	text = cleanBasic (text)
	text = text.replace ('\n', ' ')
	text = text.replace ('\t', ' ')
	for i, j in weirdChars: text = text.replace (i, j)
	text = text.strip()
	while "  " in text: text = text.replace ("  ", " ")
	tagCleanSpaces =( '{', '}', '/*', '*/', ':', ';' )
	for tag in tagCleanSpaces:
		text = text.replace (' '+ tag +' ', tag)
		text = text.replace (' '+ tag, tag)
		text = text.replace (tag +' ', tag)
		text = text.replace ('}', ';}')
		text = text.replace (';;', ';')
		text = text.replace ('};', '}')
	# nettoyer les commentaires
	commList = text.split ('/*')
	commRange = range (commList, 1)
	for c in commRange:
		f= commList[c].find ('*/')
		if '{' in commList[c][:f]: commList[c] = commList[c][f:]
	text = '/*'.join (commList)
	text = text.replace ('/**/', "")
	# optimiser les blocs
	commList = text.split ('{')
	commRange = range (commList, 1)
	for c in commRange:
		f= commList[c].find ('}')
		if f>0:
			innerComm = commList[c][:f].replace (' ', ';')
			if innerComm.count (';') >1:
				innerComm = innerComm.replace (';', ';\n\t')
				innerComm = '\n\t' + innerComm +'\n'
			elif innerComm.count (';') ==1: innerComm = ' '+ innerComm +' '
			commList[c] = innerComm + commList[c][f:]
		else: commList[c] = '\n\t' + commList[c]
	text = ' {'.join (commList)
	text = text.replace ('}', '}\n')
	text = text.replace ('/*', '\n/* ')
	text = text.replace ('*/', ' */\n')
	while '\n\n' in text: text = text.replace ('\n\n', '\n')
	return text

def cleanSql (text):
#	if '\nselect ' not in text or ';\n/\n' in text: return text
	if '\nselect ' not in text: return text
	textList = text.split ('\nselect ')
	textRange = range (1, len (textList))
	for t in textRange: textList[t] = textList[t].replace (';\n', ';\n/\n', 1)
	text = '\ncode\nselect '.join (textList)
	if '\nwith ' in text:
		textList = text.split ('\nwith ')
		textRange = range (1, len (textList))
		for t in textRange:
			d= textList[t].find ('\n')
			if ' as (' in textList[t][:d] or ' as(' in textList[t][:d]:
				f= textList[t].find (';\n')
				textList[t] = textList[t][:f].replace ('\ncode\n', '\n') + textList[t][f:]
				textList[t-1] = textList[t-1] + '\ncode'
		text = '\nwith '.join (textList)
	return text

def protectUrl (text):
	if ': C:\\' in text:
		paragraphList = text.split (': C:\\')
		paragraphRange = range (1, len (paragraphList))
		for i in paragraphRange:
			e= findEndUrl (paragraphList[i])
			if e> len (paragraphList[i]) or paragraphList[i][e] == '\n':
				d=1+ paragraphList[i-1].rfind ('\n')
				temp = paragraphList[i-1][d:]
				paragraphList[i-1] = paragraphList[i-1][:d]
				temp = '\ncode\n' + temp.capitalize()
				paragraphList[i-1] = paragraphList[i-1] + temp
				paragraphList[i] = paragraphList[i].replace ('\n', '\n/\n', 1)
		text = ': C:\\'.join (paragraphList)
	if '\nC:\\' in text:
		paragraphList = text.split ('\nC:\\')
		paragraphRange = range (1, len (paragraphList))
		for i in paragraphRange:
			e= findEndUrl (paragraphList[i])
			if e> len (paragraphList[i]) or paragraphList[i][e] == '\n':
				paragraphList[i-1] = paragraphList[i-1] + '\ncode\n'
				paragraphList[i] = paragraphList[i].replace ('\n', '\n/\n', 1)
		text = '\nC:\\'.join (paragraphList)
	if ': http' in text:
		paragraphList = text.split (': http')
		paragraphRange = range (1, len (paragraphList))
		for i in paragraphRange:
			e= findEndUrl (paragraphList[i])
			if e> len (paragraphList[i]) or paragraphList[i][e] == '\n':
				d=1+ paragraphList[i-1].rfind ('\n')
				temp = paragraphList[i-1][d:]
				paragraphList[i-1] = paragraphList[i-1][:d]
				temp = '\ncode\n' + temp.capitalize()
				paragraphList[i-1] = paragraphList[i-1] + temp
				paragraphList[i] = paragraphList[i].replace ('\n', '\n/\n', 1)
		text = ': http'.join (paragraphList)
	if '\nhttp' in text:
		paragraphList = text.split ('\nhttp')
		paragraphRange = range (1, len (paragraphList))
		for i in paragraphRange:
			e= findEndUrl (paragraphList[i])
			if e> len (paragraphList[i]) or paragraphList[i][e] == '\n':
				paragraphList[i-1] = paragraphList[i-1] + '\ncode\n'
				paragraphList[i] = paragraphList[i].replace ('\n', '\n/\n', 1)
		text = '\nhttp'.join (paragraphList)
	return text

def cleanText (text):
	text = cleanBasic (text)
	# la ponctuation
	punctuation = '?!;.:,'
	for p in punctuation: text = text.replace (' '+p, p)
	while '....' in text: text = text.replace ('....', '...')
	for letter in letters:
		text = text.replace (letter +'!', letter +' !')
		text = text.replace (letter +'?', letter +' ?')
	#	text = text.replace (letter +';', letter +' ;')
		text = text.replace ('...' + letter, '... '+ letter)
	while "  " in text: text = text.replace ("  ", " ")
	# restaurer les url
	if 'http' in text and '?' in text:
		"""
		textList = text.split ('http')
		textRange = range (1, len (textList))
		for t in textRange:
			print ('t',t)
			f= textList[t].find ('\n')
			if '\t' in textList[t][:f]: f= textList[t].find ('\t')
			if " ?" in textList[t][:f]: print (textList[t][:f])
		"""
		textList = text.split (' ?')
		textRange = range (len (textList) -1)
		for t in textRange:
			if 'http' in textList[t]:
				d= textList[t].rfind ('http')
				if " " in textList[t][d:] or '\n' " " in textList[t][d:]: textList[t] = textList[t] +" "
			else: textList[t] = textList[t] +" "
		text = '?'.join (textList)
	# restaurer les heures
	textList = text.split (':')
	textRange = range (len (textList) -1)
	for t in textRange:
		if len (textList[t]) >1 and len (textList[t+1]) >1 and textList[t][-2] in '012345' and textList[t][-1] in '0123456789' and textList[t+1][0] in '012345' and textList[t+1][1] in '0123456789': continue
		else: textList[t+1] =" "+ textList[t+1]
	text = ':'.join (textList)
	while "  " in text: text = text.replace ("  ", " ")
	text = text.replace ('--> ', '-->')
	text = text.replace ('-->', '--> ')
	charEndUrl = '\n\t \'",;!()[]{}'
	for wordStart, wordEnd in urlWords[6:]: text = text.replace (wordStart, wordEnd)
	for wordStart, wordEnd in urlWords[:6]:	# les six premiers éléments ressemblent à des débuts de mots
		for e in charEndUrl: text = text.replace (wordStart +e, wordEnd +e)
	text = text.replace ('file:///', "")
	text = text.replace ('File:///', "")
	text = text.replace (' \n', '\n')
	text = text.replace (' \t', '\t')
	text = text.replace ('\t ', '\t')
	text = text.replace ('\n ', '\n')
	return text

def shape (text, case=""):
	""" mettre le text en forme pour simplifier sa transformation
	j'utilise une mise en forme personnelle
	addCase
		rien: je ne rajoute pas les majuscules
		rest: je supprime l'ancienne casse
		upper: je rajoute les majuscules
	"""
	text = cleanText (text)
	text = '\n'+ text +'\n'
	for char in titleChars:
		while '\n'+ 3* char in text: text = text.replace ('\n'+ 3* char, '\n'+ 2* char)
	text = cleanSql (text)
	text = protectUrl (text)
	if case: text = upperCase (text, case)
	while '\n\n' in text: text = text.replace ('\n\n', '\n')
	text = text.strip()
	textList = text.split ('\n')
	titleCharsList = list (titleChars)
	textRange = range (len (titleCharsList))
	for c in textRange: titleCharsList[c] = titleCharsList[c] + titleCharsList[c] +" "
	textRange = range (len (textList))
	for l in textRange:
		if textList[l][:3] in titleCharsList: textList[l] = textList[l] +'\n'
	text = '\n'.join (textList)
	for char in titleChars:
		text = text.replace ('\n'+ 2* char +'\n', '\n\n'+ 2* char +'\n\n')
		text = text.replace ('\n'+ 2* char, '\n\n'+ 2* char)
	while '\n\n\n' in text: text = text.replace ('\n\n\n', '\n\n')
	text = text.replace ('\n/\n', '\n')
	text = text.replace ('\ncode\n', '\n')
	return text

def toMarkdown (text):
	text = shape (text, 'upper')
	text = text.replace ('== ', '= ')
	text = text.replace ('** ', '== ')
	text = text.replace ('-- ', "''' ")
	text = text.replace ('\n\t', '\n- ')
	while '\n\n' in text: text = text.replace ('\n\n', '\n')
	text = text.replace ('\n', '\n\n')
	return text

	# ________________________ fonctions de bases ________________________

def fromModel (text, model):
	# remplacer scanf
	# préparer le modèle
	model = model.lower()
	text = text.lower()
	if '</' in model:
		text = cleanHtml (text)
		model = cleanHtml (model)
	else:
		text = cleanText (text)
		model = cleanText (model)
	"""
	text = text.replace ('\n', " ")
	text = text.replace ('\t', " ")
	text = text +" "
	while "  " in text: text = text.replace ("  ", " ")
	model = model.replace ('\n', " ")
	model = model.replace ('\t', " ")
	model = model +" "
	while "  " in model: model = model.replace ("  ", " ")
	"""
	model = model.replace ('%%', '$')
	# récupérer les éléments
	modelList = model.split ('%')
	if not modelList[-1] or 1== len (modelList[-1]): trash = modelList.pop (-1)
	if not modelList[0]: trash = modelList.pop (0)
	modelList[0] = 's'+ modelList[0]
	results =[]
	for bloc in modelList:
		d= text.find (bloc[1:])
		if d==0 and text.count (bloc[1:]) >1: d= text.find (bloc[1:], 1)
		if d>0:
			results.append (text[:d])
			# convertir les éléments récupérés en chiffre
			if bloc[0] =='d': results[-1] = int (results[-1])
			elif bloc[0] =='f': results[-1] = float (results[-1])
		d= d+ len (bloc) -1
		text = text[d:]
	if text: results.append (text)
	return results

def fromModel_va (text, model):
	# remplacer scanf
	# préparer le modèle
	model = model.lower()
	text = text.lower()
	if '</' in model:
		text = cleanHtml (text)
		model = cleanHtml (model)
	else:
		text = cleanText (text)
		model = cleanText (model)
	typeList = ['d', 'f', 's']
	model = model.replace ('%%', '$')
	modelTmp = model
	for t in typeList: modelTmp = modelTmp.replace ('%'+t, '%')
	# récupérer les données du message sous forme de string
	modelList = modelTmp.split ('%')
	if not modelList[-1]: trash = modelList.pop (-1)
	if not modelList[0]: trash = modelList.pop (0)
	for line in modelList:
		if text.count (line) ==1 or '%%': text = text.replace (line, '%%', 1)
		elif '%%' in text:
			d= text.rfind ('%%')
			textTmp = text[d:].replace (line, '%%', 1)
			text = text[:d] + textTmp
		else: text = text.replace (line, '%%', 1)
	results = text.split ('%%')
	if not results[-1]: trash = results.pop (-1)
	if not results[0]: trash = results.pop (0)
	d=0
	rangeRes = range (len (results))
	for r in rangeRes:
		d= model.find ('%', d) +1
		if d==0: continue
		if model[d] =='d': results[r] = int (results[r])
		elif model[d] =='f': results[r] = float (results[r])
	if (len (results) >2+ modelTmp.count ('%')): print ('erreur: %=', modelTmp.count ('%'), 'item =', len (results))
	return results

def find (text, word, pos=0):
	posFind =-1
	if pos <0: pos = len (text) -pos
	if word in text: posFind = str.find (text, word, pos)
	elif '"' in word:
		word = word.replace ('"', "'")
		posFind = str.find (text, word, pos)
	elif "'" in word:
		word = word.replace ("'", '"')
		posFind = str.find (text, word, pos)
	return posFind

def rfind (text, word):
	posFind =-1
	if word in text: posFind = str.rfind (text, word)
	elif '"' in word:
		word = word.replace ('"', "'")
		posFind = str.rfind (text, word)
	elif "'" in word:
		word = word.replace ("'", '"')
		posFind = str.rfind (text, word)
	return posFind

def sliceWord (text, wordStart, wordEnd):
	res =""
	if wordStart in text and wordEnd in text:
		d= text.find (wordStart) + len (wordStart)
		f= text.find (wordEnd, d)
		if f>0 and f>d: res = text[d:f]
	return res

def test():
	text = """nfznvvz
zef,z "ofk" v,sknvdzkl.fnz fk" g
à 50:30:60
adresse: http://www.fr
"""
	model = """fznvvz
%ssknvdzkl.fnz %s
"""
	text = cleanText (text)
	print ('clean\t', text)
	text = upperCase (text, 'upper')
	print ('upperCase\t', text)
	print ('find\t', find (text, "fk'", 20))
	print ('rfind\t', rfind (text, "fk'"))
	textList = fromModel (text, model)
	print (textList)