From 84acd2f1b2cfc040700b90d81e189372eaf1c4c2 Mon Sep 17 00:00:00 2001 From: Tang1705 <17301138@bjtu.edu.cn> Date: Wed, 1 Jul 2020 22:57:04 +0800 Subject: [PATCH] add smile detection --- Class/detection/utils/__init__.py | 0 .../utils/__pycache__/__init__.cpython-37.pyc | Bin 0 -> 127 bytes .../utils/__pycache__/datasets.cpython-37.pyc | Bin 0 -> 5011 bytes .../utils/__pycache__/grad_cam.cpython-37.pyc | Bin 0 -> 5868 bytes .../__pycache__/inference.cpython-37.pyc | Bin 0 -> 1920 bytes .../__pycache__/preprocessor.cpython-37.pyc | Bin 0 -> 1012 bytes Class/detection/utils/data_augmentation.py | 235 ++++++++++++++++++ Class/detection/utils/datasets.py | 149 +++++++++++ Class/detection/utils/grad_cam.py | 191 ++++++++++++++ Class/detection/utils/inference.py | 48 ++++ Class/detection/utils/preprocessor.py | 29 +++ Class/detection/utils/visualizer.py | 177 +++++++++++++ Class/detection/video_emotion_color_demo.py | 105 ++++++++ 13 files changed, 934 insertions(+) create mode 100644 Class/detection/utils/__init__.py create mode 100644 Class/detection/utils/__pycache__/__init__.cpython-37.pyc create mode 100644 Class/detection/utils/__pycache__/datasets.cpython-37.pyc create mode 100644 Class/detection/utils/__pycache__/grad_cam.cpython-37.pyc create mode 100644 Class/detection/utils/__pycache__/inference.cpython-37.pyc create mode 100644 Class/detection/utils/__pycache__/preprocessor.cpython-37.pyc create mode 100644 Class/detection/utils/data_augmentation.py create mode 100644 Class/detection/utils/datasets.py create mode 100644 Class/detection/utils/grad_cam.py create mode 100644 Class/detection/utils/inference.py create mode 100644 Class/detection/utils/preprocessor.py create mode 100644 Class/detection/utils/visualizer.py create mode 100644 Class/detection/video_emotion_color_demo.py diff --git a/Class/detection/utils/__init__.py b/Class/detection/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Class/detection/utils/__pycache__/__init__.cpython-37.pyc b/Class/detection/utils/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..70bd699aeda30e0dec8f007efdd332aff139d7e6 GIT binary patch literal 127 zcmZ?b<>g`kfAOZ#$feZ&AE@lA|DGb33nv8xc8Hzx{2;!H9vsFxJacWU< zOi5vkOKNd;Nq#|$b68AiNoG!QOniK1US>&ryk0@&Ee@O9{FKt1R6CFfpMjVG01T-e AsQ>@~ literal 0 HcmV?d00001 diff --git a/Class/detection/utils/__pycache__/datasets.cpython-37.pyc b/Class/detection/utils/__pycache__/datasets.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d83af076d760d2de70a635297b9861242ee3237 GIT binary patch literal 5011 zcmbtY&2JmW72oe&eu|P6Cr*u&XoEBiQaN8JQrB?fTC(G`(IIZ)W+7v<<}9g|mb=u< zGNOp(p|yK!4?VWXfjS20A5rwye_^gIPzdO?mt5N4n_W_}lU&**_RX6&pZn(bK88PQ zwdxw4zyA9ldppl-+TW?Nd@L~U;T0T&X-rSFk?!j{@kV0!rYbGpMrkI_$o1W-&GS9g zR`Y9jHD)pUiN@@x{@C;z%wg^m&2O@GR%7)iy5C|Oti{^k+UycrV;yj7%($=hHqWrG z*3+e%WFZ@c`7ZHde=-;(QJ>nDj|t{Iyy7}Yr1?72e1qxO;Tp4Wf(1@yqocQw|e3cd(<69DU0~G^xl#db=v*XIC}&8=PpQ}lur+p2Hx9v z#Q>zx3oXzJ15__eP;dwM;KLe!xs;|iA;EDS8vI90^-Et@5)+d5nMEVh7nQ_@#Al9@ zc)7{k$NFzD7gXWxavQ77UfSBaR(+MYM%xVfJR2)+3r8Z|N+YxHUhG&2uT|w%UwkgD zpKtHnnXO;!Mv`l@ckZSAjPt0UcbEE0!%_5lcbr7fW;ag9lf0|+h?A(M>{+r|^$w`_ zd2Jl#`?4O7aD;(~XOXP!O#9K8&If}m5hVjopyv%DO^`1&zJ}tfS8re6`&dL=?Bz#$ z+mSfTv+>^TPxdBxoQS=p7PiJG(hGt(jq@O&;o?OQ&2)77)phUUYZ&@$bNd`qTg!oc zod)5d!(AQlO;lEgw^?Wl{ZyOl=_^WR;ILkW%uuq8)sbF@%p_T%LAM$LHvLNS^d_SOXS2`+5hkAI!C|3V--GUtNKs$}j3 znO(`e3o@s$3;ocVyScG23+Ggyd*Ccpvrcu?YvAp|Exc1>URQj*s1Xk%tE<&5Di;nk zZ4I?~WBPjT7UqFRdwZ;JYY$(XH_^9Q?Y31k7PX>v3V(U@mj_y2&l^RfupS$a>6B`_ zXXLb$jmg?zk|Y7)7TPCVoD=Dku0z%bVIO#gp+s_BMEwkBb1v#NIbldaS{mtC+ObH( zR5p@q2;Yqk`{~mxin|-xYt&A@GqS& z224a^KTDW&E1U4wOLHSb6hX*&c;dIqjR(p@MuL-(@b7^5YZs;(h2qe^{I%~1l>E{p zJxsIXG^nN`p|n&}L=osdUa<)> zgn>S6Dno^t{=qo1?w8IJq1lYnyk|&b>kVnEIiFGfSM?hNxqdbp-@HW=5@a21_QS7h zco{ctBpJ{vZf>2A{7UaidxE9jMeM7YhI!wE%xGLBSy;NPbKjc7WCgsHs3&f_Fo~w+`8cKD+X*s$AifrlQ!O@cIf4dgXKV+$faqg?BTjALQl& zsltIm<-_ZHK;nDpd+$8Zj$Q^T zjlu-3Jwh&Qpa%ciJZKeOQ7h^LlNrC)=Z#8Y3}7dyH>b0_O{1_@iZQKuJ6~IL3ebaf zV0sO;jbcsBU{D==qiEgG)E+JsZP-Ak=#X{&7I9?0e$qtZv<`0LftI#)jef_APqoze z6{1VgELz3Jz__6uEwJC_VzWS|ih0RrHSh{ID$&MqAGf$Z~j`RHzWtK``tb;60OYAAI z+OvpbbS+9J@M&Ql@prMFbm9@ATz2k#_|eX-?cnZ5w?4gp`_?aZ_>ZaCdxZ2!c|pQX z?^21)=Ed#F24-CPXat!!iBupGzB7P_kGNkKlSd_s^{HJsdFd@#S)&;gs-<&MW?arx zm7J`c6VQCsT{&kphgw5cPP~j4Ew%P?$9_{)D|_%86{9ptx!QRi!pSXtfLBoROeu(G zSo#Yn+xk_5ocadb`UYwp<=C610b6mvle7OO_pZ_)Q-$VTPAA-}QNl;x9R$dgf>FjM z&{H6rL2xt)ld8oH0+#iIfUjc=zf6R}jWS-f%L`VmzJbyxPB@|%qB1DjIzD0Dt9hH= znrd4aF2QVNxSya>lsW!YW)RHjqC=M)R{O%ePi=Lii(7QfVXZIV>LF=E@&R43aQWK6 zWowf?SN6^|c6pWgKfkMTM`B3+2cDzh?DklHbRDCf-JKh8l_TJY24bC#`Zmwcck(D zEot1R_hV`7?#wptrJsjM%$~X|z$#l9jY6gNPeVl3$tsiZ5yD)IUV;>_0Ra@~%XEFy z;5lUK%l}V}(`hs+jW?CXpFW=!oewnLUTJ)-)cCpTIPH?JRL2jfN~(i?9}(Xt!iYpf z21JHL_KA>wq0>X+6C#8!3>^`l63K{+i5wB33qBVh|ATV*U5uOHU1^oB04uFh>p%bW zM|%GD(>r|=ReA`>gnSi+ehG%65Z!hVCn(R?ah<(d#tr)w87|GM-1sY!9Oo!D$=(H6rg*v&Wx9Av+3J z>LwqkOu?_SEU5l)@SnaB(KLP=n8!UZ9ai0yD^xPSu8Kf)Te~o?TK*Y$uBpY;%}iaW zevQpk+9Z6^uS`t&30hV`8jmqVlL`)KxFlMNPlMqJo?(S#znQGJ*GWPFjRN zAf&+<2%&dSrLx!YJ<*THCtGo*?oG&nk=ccUzl~ODW8dSGGD~!DJ!09Y1k63OR)IHr tbb$XWl!GoE2jN^Dk6QPeJiYX8l!4Ule*rEx=T!gz literal 0 HcmV?d00001 diff --git a/Class/detection/utils/__pycache__/grad_cam.cpython-37.pyc b/Class/detection/utils/__pycache__/grad_cam.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c61df1d56852c1d3852e9162f8718b0d3eea23d GIT binary patch literal 5868 zcmai2Npl;=8J(UD0}xzINu*XvvKB}dC{dA|D2`-XvYo}IZP~G16S^iNbQ2tKu&8Gs zk<1{K3hlCTaK*W#D#?X7`Um8YYko$bQe|IsOr=tl^2K>y4*)L8GN|c&dinbG`(ESK z@$r&@=db_#dgbg3-NSw#-BmjX4h_n10Kfi7Eng6_ao*VOqe~w@9 z&x?{czs>v$;)pnh_@cP5QxO-@>XIn$To#v*dO;i)7sLzv;9(F}5 zjl!66aXaKev&{wNtac}PifuBGp6W=^k?opDk|1hzx@nCn=rQryc<HjP00x%$FOOg|)n8%FAlH-ASWnv?XNlSVZ-;G^tq1Uc1rRRQBspER<7= z+li={YC_K^SVdtgY@$Nvzcl0o2K(mX-CHX^O@vHV(uXVeM6#Z?J1ciTSm~xwoUGI( z)>8|cRdjnqibSNrGviLu#9agiD>IkLDWqx^Rs!skviitj;pCj~`aW zmMgY0(^cg)yKx$I;!VYVq6+Pn2-eyumc};JL@i83y)8FkK`u7bSgYF%YH^q(BGHS< z1DcPTO+#77Y)7BK7|9{jBxG-c$vK3hJ)pgySI?3*BO^<*W!)imZ%l7lkgSCrad4#5 z|7oP4-A%E{`4{u(a5yZ^^ZsGI;jpyznKlnhWa70lqtEEa*%9LyO~_oX|9$PNCx8Ce z*S9OSd=4KdJ9#KmWj96GQf~6F8%j}Ok`{V!p2b#MHp3W*If2@IKBlBm_nBb?MVe`9 zG&B^s`MKFMQZqBQSuZz^wJZnFw!@8xz-P8PYD{c&!WIvB)J!DpdxiO9Q8TQIAe1uP zJUFTRz~}~NXjVClCYZ-0<{uQ`38ouW6CQr0?NPZXw8w2D+R_mpwCMIXbzyZWBIdv2faqyv1&5 z&&n))9KT5;8E+fxLoPd9`-y(Gm|{9BlyYGfhM9=bK27 zd0~2@KI~+}>{+SRurutd#-5#8I}Tnqv-_GfxHWI|sHBjY2)({hH#4UW(!`e7%TEa` znDQw^IZlhp(YvCGL<@1aDWr1qQ<|S7s~K*faCQw?7r4`cX(t2AvMLLSx!5>wyMrph_5GnD!E!MbWA~b0p?#?PdpD9Sj=; z!#ONrz(h*}&}^Aau)|D#A7vx5B}^3YYODSl2_oCRK|tLxIV>B&w!eJA$(}!s);nmFQRpn^w9)RRGrI%uy1+wDpJLq0x(KSrAbry~e96< zSF7K{dJtZ)I!6V5ecSt{gppKfU`^j`&2rOr4f;nb8T;HNV&rkMpG+*c>%^U=N6V6Y|9d zIDg8TvP;1AgyjL@9?N$3t_SlwQiLb@T{e2C%Lt_SdY@R?t?0}a&fh`){bM5F6s?gd=R%E zEH>dMgj^IGVY3s9WRbXc~uqbTW;3)g)^lj(ltBJ3~{2K7Jwfu6tI891cB-O>Cf ziE*EPivTKXtzzYQI*YnmIo+t0eiyh}peA+;4qO$WCxn2)JWgd+l5%*lIO^nlIg`6} zGz{7jCkE80ia0T?qyyclTr305@4twVNxrLwNgdqB2ssSw308tzn_?b7KtL$75}Rdm z1uaHIM2nk03R+bDGm&U`A#&KzQ@UZ`fzXXKdBfgy;3(PJ8$e1;r&^|1HF~dYAkayw-3^rC(10Kq&#fd7)NeGyC63! z)S(%AUgiN5g=KB$eek8@Mba%P>v?C!6XOZnCM5w4oa@qo(JH1S#Vn6dpf!P(1Kvxq zzJ1(oq5L0sjjhW=pD{Z8L!YsiQ9uGxeusja6kMg?MFf>1gdbgW20TD3!TpH*Za}1xyU+4KJnM30lSuc+ zG~x%YnR-^!&n!!uW1oGrX-2;DnGYiLGvFB)03(+15YIqA&BBkCm=tI+(kBEklCRp6 z-URD_L%=b0bDOQ0J*X3SKe}ccpPH-YJm5CWmU4BDwI`ww4_{IPjR_X1&FMu*=@?}=to)bs3?M4 zMUfg0JGfciZO3i-R@jm6Q;YK5x8HpG{#&oS6a4tK_bWxosnF5nreW?sCmw}y7vu1e zWKL9JaOqao-B;f7OL-1=oVs%JG_;BcQ@GxP9`F=od953xzfpIC8$1R;XImIQSwLWb z1gpU5QDF28Xb_6ennJ#8&H=MCOg=`N5hV`kP$Tv{YDiAuzGebP7KjgiF~=`Z!hjKw z<2L=NVGizbrtVJDY{gYGyMI?;R{9+0hM9gwmC6)@$ww5BGe5Z86ML4s|C&RK{Q_SM zt}-Sq(UP?MFU_aFqOLLoHGP#pg2e5}^-E;-EYPM!!Sw@p`ceR(?qG?4p{0)kE#9QV z3ofU2F2}J$TNhUcJ9GE6bu*VB4SolcYo2GgViY*~fwAmmjhP(FKgS%PlV}%muzo

^cO<@LXhdcEzXVAcKW9d36)LiiT z!+YLMc@_h0&Fs0?VlCB1r;hVpMHQl7P_&eri`Z8F5P?6gg;13gI%K#i3=-P?_|ti1 zu4JK~q|gpZmGg3~KNIc3w1!m<2%@v8_ePac(MsA9K438Jav@bM;+J%hi0>$)V)vAWV;JPtGq0l-r4F z>-d`vJ}-o>YIj6SIeI^oO_3ydpq-?~jVfR^JgTK?Y>>dkB7w@1b-K4s4hnHt^&evS zVJ#0fKwBmU-NgPqQH7}aSVqv6yZ<@tRT}uR8;jc{yAu-Pgd&n;`A?_KvgNs+jh}k literal 0 HcmV?d00001 diff --git a/Class/detection/utils/__pycache__/inference.cpython-37.pyc b/Class/detection/utils/__pycache__/inference.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..21d688bb889def4ac8fcd9dfe0930b09f91e152e GIT binary patch literal 1920 zcmZ8iO>f;q6!rM~eWXd+(6j|a7iD3oq^N8FRge-AHBxyXS`b)IwH(j9_e}k4GoCa_ z3im!d;P`AO5Z~H4aoE28w^=xj(4(n_;_ONCb{yc~S%w6Ly_g=9$fqGKVHm$R7`N2er9@Mp>t< zwOZENOK0I=WVgqDfShq*A6QksJ5T?Ma=gneI(7*JNoS z^gyL@Dpbl3pH}IlfAgF9bT4tO^TF{gxZtUd?%euL}cs!yyBwh zFt2ineqFx?Z}q{?2_b^ zUO6bL1H!sbbZK7-!(iA5wO`hUN(Yll9;K2)8LK!DuMy`Rxt%7oOIKWehGF*t$t2B0 zL;N(P1B+c{A?q`F9rn3EiDs&X%lj}|YwRwZg}d+;z7Z>Hoz|T|rmeSWwQlQgS{UJB zmL7$Z;rcbbXk;5~x+lR=T22cQ1$vE~NV2MuJTFrv8a?PlY&5OCtSTz0*|BCPnjL92 zy9{QX%k(HYs18e9Fokh^3*S0Z=8(zT7@ZSJIM2iFV+;%$yTjA6vAysYL2I}E4GXOc zJU|0v0mdD^0O(v3j>eHFls9pW4ywrnv^4S^jO4pC%y$iCGX*QL%Orr5R}L}u;eD1%q6!L%zU9cwjlp@SGoWd*pWf1d!HKajf*l-q)k2$)nwGpFG z{n2jn@L|-~gA2S#dUow99F%X985+RAqr*4I;{K#6k%vrH>Zw_tJugLb;n6O?OV5x? zcfOA+JDJ*R>^gIq`~WlmOF8E9~^ zjHXXXxLYFU&nUh{`STO{sJ%A=2joQ85ak^+s~t1z^Hd>2sv=SC-2_M{F>=aya=~#@oHWTZe1M#KScuQa W!G@AVXyiF``d;XVVTjgZ{op?!{l*6X literal 0 HcmV?d00001 diff --git a/Class/detection/utils/__pycache__/preprocessor.cpython-37.pyc b/Class/detection/utils/__pycache__/preprocessor.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7cbe85f801f508a9be4a40705fdfc9bf6e52bde8 GIT binary patch literal 1012 zcmZ8g&2AGh5cb&XWRs>zRS+sd;)oQ4IJ7rZK}ES#LRBRShgC#m#ZK9>n_cV;B4~S} zSKc8#@=88c;?!3_9GLO8rL84d2tdN^@(JKAa$uut*Y$UEJ;|9yMDcjx#txP4h<#CDdWYj zai6QA9PNG>&8swXqtcqv78B!KVV{=^zxLlTPG{x3YGDl5qr}jK7$V#;SYmqu-6EY;G)|h@X^GqmI)>6x8+=>22DNmbRB60K=!L`fTQ$S+i;@9;Z;p%|Pcr44 zaeies&u<$}T*kht-*!2=K zm5T 0: + image_array, box_corners = self.horizontal_flip(image_array, + box_corners) + + if self.vertical_flip_probability > 0: + image_array, box_corners = self.vertical_flip(image_array, + box_corners) + return image_array, box_corners + + def preprocess_images(self, image_array): + return preprocess_input(image_array) + + def flow(self, mode='train'): + while True: + if mode == 'train': + shuffle(self.train_keys) + keys = self.train_keys + elif mode == 'val' or mode == 'demo': + shuffle(self.validation_keys) + keys = self.validation_keys + else: + raise Exception('invalid mode: %s' % mode) + + inputs = [] + targets = [] + for key in keys: + image_path = self.path_prefix + key + image_array = imread(image_path) + image_array = imresize(image_array, self.image_size) + + num_image_channels = len(image_array.shape) + if num_image_channels != 3: + continue + + ground_truth = self.ground_truth_data[key] + + if self.do_random_crop: + image_array = self._do_random_crop(image_array) + + image_array = image_array.astype('float32') + if mode == 'train' or mode == 'demo': + if self.ground_truth_transformer is not None: + image_array, ground_truth = self.transform( + image_array, + ground_truth) + ground_truth = ( + self.ground_truth_transformer.assign_boxes( + ground_truth)) + else: + image_array = self.transform(image_array)[0] + + if self.grayscale: + image_array = cv2.cvtColor( + image_array.astype('uint8'), + cv2.COLOR_RGB2GRAY).astype('float32') + image_array = np.expand_dims(image_array, -1) + + inputs.append(image_array) + targets.append(ground_truth) + if len(targets) == self.batch_size: + inputs = np.asarray(inputs) + targets = np.asarray(targets) + # this will not work for boxes + targets = to_categorical(targets) + if mode == 'train' or mode == 'val': + inputs = self.preprocess_images(inputs) + yield self._wrap_in_dictionary(inputs, targets) + if mode == 'demo': + yield self._wrap_in_dictionary(inputs, targets) + inputs = [] + targets = [] + + def _wrap_in_dictionary(self, image_array, targets): + return [{'input_1': image_array}, + {'predictions': targets}] diff --git a/Class/detection/utils/datasets.py b/Class/detection/utils/datasets.py new file mode 100644 index 0000000..aa445a7 --- /dev/null +++ b/Class/detection/utils/datasets.py @@ -0,0 +1,149 @@ +from scipy.io import loadmat +import pandas as pd +import numpy as np +from random import shuffle +import os +import cv2 + + +class DataManager(object): + """Class for loading fer2013 emotion classification dataset or + imdb gender classification dataset.""" + def __init__(self, dataset_name='imdb', + dataset_path=None, image_size=(48, 48)): + + self.dataset_name = dataset_name + self.dataset_path = dataset_path + self.image_size = image_size + if self.dataset_path is not None: + self.dataset_path = dataset_path + elif self.dataset_name == 'imdb': + self.dataset_path = '../datasets/imdb_crop/imdb.mat' + elif self.dataset_name == 'fer2013': + self.dataset_path = '../datasets/fer2013/fer2013.csv' + elif self.dataset_name == 'KDEF': + self.dataset_path = '../datasets/KDEF/' + else: + raise Exception( + 'Incorrect dataset name, please input imdb or fer2013') + + def get_data(self): + if self.dataset_name == 'imdb': + ground_truth_data = self._load_imdb() + elif self.dataset_name == 'fer2013': + ground_truth_data = self._load_fer2013() + elif self.dataset_name == 'KDEF': + ground_truth_data = self._load_KDEF() + return ground_truth_data + + def _load_imdb(self): + face_score_treshold = 3 + dataset = loadmat(self.dataset_path) + image_names_array = dataset['imdb']['full_path'][0, 0][0] + gender_classes = dataset['imdb']['gender'][0, 0][0] + face_score = dataset['imdb']['face_score'][0, 0][0] + second_face_score = dataset['imdb']['second_face_score'][0, 0][0] + face_score_mask = face_score > face_score_treshold + second_face_score_mask = np.isnan(second_face_score) + unknown_gender_mask = np.logical_not(np.isnan(gender_classes)) + mask = np.logical_and(face_score_mask, second_face_score_mask) + mask = np.logical_and(mask, unknown_gender_mask) + image_names_array = image_names_array[mask] + gender_classes = gender_classes[mask].tolist() + image_names = [] + for image_name_arg in range(image_names_array.shape[0]): + image_name = image_names_array[image_name_arg][0] + image_names.append(image_name) + return dict(zip(image_names, gender_classes)) + + def _load_fer2013(self): + data = pd.read_csv(self.dataset_path) + pixels = data['pixels'].tolist() + width, height = 48, 48 + faces = [] + for pixel_sequence in pixels: + face = [int(pixel) for pixel in pixel_sequence.split(' ')] + face = np.asarray(face).reshape(width, height) + face = cv2.resize(face.astype('uint8'), self.image_size) + faces.append(face.astype('float32')) + faces = np.asarray(faces) + faces = np.expand_dims(faces, -1) + emotions = pd.get_dummies(data['emotion']).as_matrix() + return faces, emotions + + def _load_KDEF(self): + class_to_arg = get_class_to_arg(self.dataset_name) + num_classes = len(class_to_arg) + + file_paths = [] + for folder, subfolders, filenames in os.walk(self.dataset_path): + for filename in filenames: + if filename.lower().endswith(('.jpg')): + file_paths.append(os.path.join(folder, filename)) + + num_faces = len(file_paths) + y_size, x_size = self.image_size + faces = np.zeros(shape=(num_faces, y_size, x_size)) + emotions = np.zeros(shape=(num_faces, num_classes)) + for file_arg, file_path in enumerate(file_paths): + image_array = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE) + image_array = cv2.resize(image_array, (y_size, x_size)) + faces[file_arg] = image_array + file_basename = os.path.basename(file_path) + file_emotion = file_basename[4:6] + # there are two file names in the dataset + # that don't match the given classes + try: + emotion_arg = class_to_arg[file_emotion] + except: + continue + emotions[file_arg, emotion_arg] = 1 + faces = np.expand_dims(faces, -1) + return faces, emotions + + +def get_labels(dataset_name): + if dataset_name == 'fer2013': + return {0: 'angry', 1: 'disgust', 2: 'fear', 3: 'happy', + 4: 'sad', 5: 'surprise', 6: 'neutral'} + elif dataset_name == 'imdb': + return {0: 'woman', 1: 'man'} + elif dataset_name == 'KDEF': + return {0: 'AN', 1: 'DI', 2: 'AF', 3: 'HA', 4: 'SA', 5: 'SU', 6: 'NE'} + else: + raise Exception('Invalid dataset name') + + +def get_class_to_arg(dataset_name='fer2013'): + if dataset_name == 'fer2013': + return {'angry': 0, 'disgust': 1, 'fear': 2, 'happy': 3, 'sad': 4, + 'surprise': 5, 'neutral': 6} + elif dataset_name == 'imdb': + return {'woman': 0, 'man': 1} + elif dataset_name == 'KDEF': + return {'AN': 0, 'DI': 1, 'AF': 2, 'HA': 3, 'SA': 4, 'SU': 5, 'NE': 6} + else: + raise Exception('Invalid dataset name') + + +def split_imdb_data(ground_truth_data, validation_split=.2, do_shuffle=False): + ground_truth_keys = sorted(ground_truth_data.keys()) + if do_shuffle is not False: + shuffle(ground_truth_keys) + training_split = 1 - validation_split + num_train = int(training_split * len(ground_truth_keys)) + train_keys = ground_truth_keys[:num_train] + validation_keys = ground_truth_keys[num_train:] + return train_keys, validation_keys + + +def split_data(x, y, validation_split=.2): + num_samples = len(x) + num_train_samples = int((1 - validation_split)*num_samples) + train_x = x[:num_train_samples] + train_y = y[:num_train_samples] + val_x = x[num_train_samples:] + val_y = y[num_train_samples:] + train_data = (train_x, train_y) + val_data = (val_x, val_y) + return train_data, val_data diff --git a/Class/detection/utils/grad_cam.py b/Class/detection/utils/grad_cam.py new file mode 100644 index 0000000..28c06d9 --- /dev/null +++ b/Class/detection/utils/grad_cam.py @@ -0,0 +1,191 @@ +import cv2 +import h5py +import keras +import keras.backend as K +from keras.layers.core import Lambda +from keras.models import Sequential +from keras.models import load_model +import numpy as np +import tensorflow as tf +from tensorflow.python.framework import ops + +from .preprocessor import preprocess_input + + +def reset_optimizer_weights(model_filename): + model = h5py.File(model_filename, 'r+') + del model['optimizer_weights'] + model.close() + + +def target_category_loss(x, category_index, num_classes): + return tf.multiply(x, K.one_hot([category_index], num_classes)) + + +def target_category_loss_output_shape(input_shape): + return input_shape + + +def normalize(x): + # utility function to normalize a tensor by its L2 norm + return x / (K.sqrt(K.mean(K.square(x))) + 1e-5) + + +def load_image(image_array): + image_array = np.expand_dims(image_array, axis=0) + image_array = preprocess_input(image_array) + return image_array + + +def register_gradient(): + if "GuidedBackProp" not in ops._gradient_registry._registry: + @ops.RegisterGradient("GuidedBackProp") + def _GuidedBackProp(op, gradient): + dtype = op.inputs[0].dtype + guided_gradient = (gradient * tf.cast(gradient > 0., dtype) * + tf.cast(op.inputs[0] > 0., dtype)) + return guided_gradient + + +def compile_saliency_function(model, activation_layer='conv2d_7'): + input_image = model.input + layer_output = model.get_layer(activation_layer).output + max_output = K.max(layer_output, axis=3) + saliency = K.gradients(K.sum(max_output), input_image)[0] + return K.function([input_image, K.learning_phase()], [saliency]) + + +def modify_backprop(model, name, task): + graph = tf.get_default_graph() + with graph.gradient_override_map({'Relu': name}): + + # get layers that have an activation + activation_layers = [layer for layer in model.layers + if hasattr(layer, 'activation')] + + # replace relu activation + for layer in activation_layers: + if layer.activation == keras.activations.relu: + layer.activation = tf.nn.relu + + # re-instanciate a new model + if task == 'gender': + model_path = '../trained_models/gender_models/gender_mini_XCEPTION.21-0.95.hdf5' + elif task == 'emotion': + model_path = '../trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' + # model_path = '../trained_models/fer2013_mini_XCEPTION.119-0.65.hdf5' + # model_path = '../trained_models/fer2013_big_XCEPTION.54-0.66.hdf5' + new_model = load_model(model_path, compile=False) + return new_model + + +def deprocess_image(x): + """ Same normalization as in: + https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py + """ + if np.ndim(x) > 3: + x = np.squeeze(x) + # normalize tensor: center on 0., ensure std is 0.1 + x = x - x.mean() + x = x / (x.std() + 1e-5) + x = x * 0.1 + + # clip to [0, 1] + x = x + 0.5 + x = np.clip(x, 0, 1) + + # convert to RGB array + x = x * 255 + if K.image_dim_ordering() == 'th': + x = x.transpose((1, 2, 0)) + x = np.clip(x, 0, 255).astype('uint8') + return x + + +def compile_gradient_function(input_model, category_index, layer_name): + model = Sequential() + model.add(input_model) + + num_classes = model.output_shape[1] + target_layer = lambda x: target_category_loss(x, category_index, num_classes) + model.add(Lambda(target_layer, + output_shape=target_category_loss_output_shape)) + + loss = K.sum(model.layers[-1].output) + conv_output = model.layers[0].get_layer(layer_name).output + gradients = normalize(K.gradients(loss, conv_output)[0]) + gradient_function = K.function([model.layers[0].input, K.learning_phase()], + [conv_output, gradients]) + return gradient_function + + +def calculate_gradient_weighted_CAM(gradient_function, image): + output, evaluated_gradients = gradient_function([image, False]) + output, evaluated_gradients = output[0, :], evaluated_gradients[0, :, :, :] + weights = np.mean(evaluated_gradients, axis=(0, 1)) + CAM = np.ones(output.shape[0: 2], dtype=np.float32) + for weight_arg, weight in enumerate(weights): + CAM = CAM + (weight * output[:, :, weight_arg]) + CAM = cv2.resize(CAM, (64, 64)) + CAM = np.maximum(CAM, 0) + heatmap = CAM / np.max(CAM) + + # Return to BGR [0..255] from the preprocessed image + image = image[0, :] + image = image - np.min(image) + image = np.minimum(image, 255) + + CAM = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET) + CAM = np.float32(CAM) + np.float32(image) + CAM = 255 * CAM / np.max(CAM) + return np.uint8(CAM), heatmap + + +def calculate_guided_gradient_CAM( + preprocessed_input, gradient_function, saliency_function): + CAM, heatmap = calculate_gradient_weighted_CAM( + gradient_function, preprocessed_input) + saliency = saliency_function([preprocessed_input, 0]) + # gradCAM = saliency[0] * heatmap[..., np.newaxis] + # return deprocess_image(gradCAM) + return deprocess_image(saliency[0]) + # return saliency[0] + + +def calculate_guided_gradient_CAM_v2( + preprocessed_input, gradient_function, + saliency_function, target_size=(128, 128)): + CAM, heatmap = calculate_gradient_weighted_CAM( + gradient_function, preprocessed_input) + heatmap = np.squeeze(heatmap) + heatmap = cv2.resize(heatmap.astype('uint8'), target_size) + saliency = saliency_function([preprocessed_input, 0]) + saliency = np.squeeze(saliency[0]) + saliency = cv2.resize(saliency.astype('uint8'), target_size) + gradCAM = saliency * heatmap + gradCAM = deprocess_image(gradCAM) + return np.expand_dims(gradCAM, -1) + + +if __name__ == '__main__': + import pickle + faces = pickle.load(open('faces.pkl', 'rb')) + face = faces[0] + model_filename = '../../trained_models/emotion_models/mini_XCEPTION.523-0.65.hdf5' + # reset_optimizer_weights(model_filename) + model = load_model(model_filename) + + preprocessed_input = load_image(face) + predictions = model.predict(preprocessed_input) + predicted_class = np.argmax(predictions) + gradient_function = compile_gradient_function( + model, predicted_class, 'conv2d_6') + register_gradient() + guided_model = modify_backprop(model, 'GuidedBackProp') + saliency_function = compile_saliency_function(guided_model) + guided_gradCAM = calculate_guided_gradient_CAM( + preprocessed_input, gradient_function, saliency_function) + + cv2.imwrite('guided_gradCAM.jpg', guided_gradCAM) + + diff --git a/Class/detection/utils/inference.py b/Class/detection/utils/inference.py new file mode 100644 index 0000000..6793fdc --- /dev/null +++ b/Class/detection/utils/inference.py @@ -0,0 +1,48 @@ +import cv2 +import matplotlib.pyplot as plt +import numpy as np +from keras.preprocessing import image + + +def load_image(image_path, grayscale=False, target_size=None): + pil_image = image.load_img(image_path, grayscale, target_size) + return image.img_to_array(pil_image) + + +def load_detection_model(model_path): + # detection_model = cv2.CascadeClassifier(model_path) + detection_model = cv2.dnn.readNetFromCaffe("data/data_opencv/deploy.prototxt.txt", model_path) + return detection_model + + +def detect_faces(detection_model, gray_image_array): + blob = cv2.dnn.blobFromImage(cv2.resize(gray_image_array, (300, 300)), 1.0, + (300, 300), (104.0, 177.0, 123.0)) + detection_model.setInput(blob) + return detection_model.forward() + # return detection_model.detectMultiScale(gray_image_array, 1.3, 5) + + +def draw_bounding_box(face_coordinates, image_array, color): + x, y, w, h = face_coordinates + cv2.rectangle(image_array, (x, y), (x + w, y + h), color, 2) + + +def apply_offsets(face_coordinates, offsets): + x, y, width, height = face_coordinates + x_off, y_off = offsets + return (x - x_off, x + width + x_off, y - y_off, y + height + y_off) + + +def draw_text(coordinates, image_array, text, color, x_offset=0, y_offset=0, + font_scale=2, thickness=2): + x, y = coordinates[:2] + cv2.putText(image_array, text, (x + x_offset, y + y_offset), + cv2.FONT_HERSHEY_SIMPLEX, + font_scale, color, thickness, cv2.LINE_AA) + + +def get_colors(num_classes): + colors = plt.cm.hsv(np.linspace(0, 1, num_classes)).tolist() + colors = np.asarray(colors) * 255 + return colors diff --git a/Class/detection/utils/preprocessor.py b/Class/detection/utils/preprocessor.py new file mode 100644 index 0000000..72953f5 --- /dev/null +++ b/Class/detection/utils/preprocessor.py @@ -0,0 +1,29 @@ +import numpy as np +from PIL import Image +import numpy as np +from imageio import imread + +def preprocess_input(x, v2=True): + x = x.astype('float32') + x = x / 255.0 + if v2: + x = x - 0.5 + x = x * 2.0 + return x + + +def _imread(image_name): + return imread(image_name) + + +def _imresize(image_array, size): + return np.array(Image.fromarray(image_array).resize(size)) + # return imresize(image_array, size) + + +def to_categorical(integer_classes, num_classes=2): + integer_classes = np.asarray(integer_classes, dtype='int') + num_samples = integer_classes.shape[0] + categorical = np.zeros((num_samples, num_classes)) + categorical[np.arange(num_samples), integer_classes] = 1 + return categorical diff --git a/Class/detection/utils/visualizer.py b/Class/detection/utils/visualizer.py new file mode 100644 index 0000000..aa32550 --- /dev/null +++ b/Class/detection/utils/visualizer.py @@ -0,0 +1,177 @@ +import numpy as np +import matplotlib.cm as cm +from mpl_toolkits.axes_grid1 import make_axes_locatable +import matplotlib.pyplot as plt +import numpy.ma as ma + + +def make_mosaic(images, num_rows, num_cols, border=1, class_names=None): + num_images = len(images) + image_shape = images.shape[1:] + mosaic = ma.masked_all( + (num_rows * image_shape[0] + (num_rows - 1) * border, + num_cols * image_shape[1] + (num_cols - 1) * border), + dtype=np.float32) + paddedh = image_shape[0] + border + paddedw = image_shape[1] + border + for image_arg in range(num_images): + row = int(np.floor(image_arg / num_cols)) + col = image_arg % num_cols + image = np.squeeze(images[image_arg]) + image_shape = image.shape + mosaic[row * paddedh:row * paddedh + image_shape[0], + col * paddedw:col * paddedw + image_shape[1]] = image + return mosaic + + +def make_mosaic_v2(images, num_mosaic_rows=None, + num_mosaic_cols=None, border=1): + images = np.squeeze(images) + num_images, image_pixels_rows, image_pixels_cols = images.shape + if num_mosaic_rows is None and num_mosaic_cols is None: + box_size = int(np.ceil(np.sqrt(num_images))) + num_mosaic_rows = num_mosaic_cols = box_size + num_mosaic_pixel_rows = num_mosaic_rows * (image_pixels_rows + border) + num_mosaic_pixel_cols = num_mosaic_cols * (image_pixels_cols + border) + mosaic = np.empty(shape=(num_mosaic_pixel_rows, num_mosaic_pixel_cols)) + mosaic_col_arg = 0 + mosaic_row_arg = 0 + for image_arg in range(num_images): + if image_arg % num_mosaic_cols == 0 and image_arg != 0: + mosaic_col_arg = mosaic_col_arg + 1 + mosaic_row_arg = 0 + x0 = image_pixels_cols * (mosaic_row_arg) + x1 = image_pixels_cols * (mosaic_row_arg + 1) + y0 = image_pixels_rows * (mosaic_col_arg) + y1 = image_pixels_rows * (mosaic_col_arg + 1) + image = images[image_arg] + mosaic[y0:y1, x0:x1] = image + mosaic_row_arg = mosaic_row_arg + 1 + return mosaic + + +def pretty_imshow(axis, data, vmin=None, vmax=None, cmap=None): + if cmap is None: + cmap = cm.jet + if vmin is None: + vmin = data.min() + if vmax is None: + vmax = data.max() + cax = None + divider = make_axes_locatable(axis) + cax = divider.append_axes('right', size='5%', pad=0.05) + image = axis.imshow(data, vmin=vmin, vmax=vmax, + interpolation='nearest', cmap=cmap) + plt.colorbar(image, cax=cax) + + +def normal_imshow(axis, data, vmin=None, vmax=None, + cmap=None, axis_off=True): + if cmap is None: + cmap = cm.jet + if vmin is None: + vmin = data.min() + if vmax is None: + vmax = data.max() + image = axis.imshow(data, vmin=vmin, vmax=vmax, + interpolation='nearest', cmap=cmap) + if axis_off: + plt.axis('off') + return image + + +def display_image(face, class_vector=None, + class_decoder=None, pretty=False): + if class_vector is not None and class_decoder is None: + raise Exception('Provide class decoder') + face = np.squeeze(face) + color_map = None + if len(face.shape) < 3: + color_map = 'gray' + plt.figure() + if class_vector is not None: + class_arg = np.argmax(class_vector) + class_name = class_decoder[class_arg] + plt.title(class_name) + if pretty: + pretty_imshow(plt.gca(), face, cmap=color_map) + else: + plt.imshow(face, color_map) + + +def draw_mosaic(data, num_rows, num_cols, class_vectors=None, + class_decoder=None, cmap='gray'): + + if class_vectors is not None and class_decoder is None: + raise Exception('Provide class decoder') + + figure, axis_array = plt.subplots(num_rows, num_cols) + figure.set_size_inches(8, 8, forward=True) + titles = [] + if class_vectors is not None: + for vector_arg in range(len(class_vectors)): + class_arg = np.argmax(class_vectors[vector_arg]) + class_name = class_decoder[class_arg] + titles.append(class_name) + + image_arg = 0 + for row_arg in range(num_rows): + for col_arg in range(num_cols): + image = data[image_arg] + image = np.squeeze(image) + axis_array[row_arg, col_arg].axis('off') + axis_array[row_arg, col_arg].imshow(image, cmap=cmap) + axis_array[row_arg, col_arg].set_title(titles[image_arg]) + image_arg = image_arg + 1 + plt.tight_layout() + + +if __name__ == '__main__': + # from utils.data_manager import DataManager + from utils.utils import get_labels + from keras.models import load_model + import pickle + + # dataset_name = 'fer2013' + # model_path = '../trained_models/emotion_models/simple_CNN.985-0.66.hdf5' + dataset_name = 'fer2013' + class_decoder = get_labels(dataset_name) + # data_manager = DataManager(dataset_name) + # faces, emotions = data_manager.get_data() + faces = pickle.load(open('faces.pkl', 'rb')) + emotions = pickle.load(open('emotions.pkl', 'rb')) + pretty_imshow(plt.gca(), make_mosaic(faces[:4], 2, 2), cmap='gray') + plt.show() + + """ + image_arg = 0 + face = faces[image_arg:image_arg + 1] + emotion = emotions[image_arg:image_arg + 1] + display_image(face, emotion, class_decoder) + plt.show() + + normal_imshow(plt.gca(), make_mosaic(faces[:4], 3, 3), cmap='gray') + plt.show() + + draw_mosaic(faces, 2, 2, emotions, class_decoder) + plt.show() + + """ + model = load_model('../trained_models/emotion_models/simple_CNN.985-0.66.hdf5') + conv1_weights = model.layers[2].get_weights() + kernel_conv1_weights = conv1_weights[0] + kernel_conv1_weights = np.squeeze(kernel_conv1_weights) + kernel_conv1_weights = np.rollaxis(kernel_conv1_weights, 2, 0) + kernel_conv1_weights = np.expand_dims(kernel_conv1_weights, -1) + num_kernels = kernel_conv1_weights.shape[0] + box_size = int(np.ceil(np.sqrt(num_kernels))) + print('Box size:', box_size) + + print('Kernel shape', kernel_conv1_weights.shape) + plt.figure(figsize=(15, 15)) + plt.title('conv1 weights') + pretty_imshow( + plt.gca(), + make_mosaic(kernel_conv1_weights, box_size, box_size), + cmap=cm.binary) + plt.show() diff --git a/Class/detection/video_emotion_color_demo.py b/Class/detection/video_emotion_color_demo.py new file mode 100644 index 0000000..aa115a6 --- /dev/null +++ b/Class/detection/video_emotion_color_demo.py @@ -0,0 +1,105 @@ +from statistics import mode + +import cv2 +from keras.models import load_model +import numpy as np + +from utils.datasets import get_labels +from utils.inference import detect_faces +from utils.inference import draw_text +from utils.inference import draw_bounding_box +from utils.inference import load_detection_model +from utils.preprocessor import preprocess_input + +# parameters for loading data and images +detection_model_path = './data/data_opencv/res10_300x300_ssd_iter_140000.caffemodel' +emotion_model_path = 'data/trained_models/emotion_models/fer2013_mini_XCEPTION.102-0.66.hdf5' +emotion_labels = get_labels('fer2013') + +# hyper-parameters for bounding boxes shape +frame_window = 10 +emotion_offsets = (20, 40) + +# loading models +face_detection = load_detection_model(detection_model_path) +emotion_classifier = load_model(emotion_model_path, compile=False) + +# getting input model shapes for inference +emotion_target_size = emotion_classifier.input_shape[1:3] + +# starting lists for calculating modes +emotion_window = [] + +# starting video streaming +cv2.namedWindow('window_frame') +video_capture = cv2.VideoCapture(0) +while True: + bgr_image = video_capture.read()[1] + gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY) + rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) + faces = detect_faces(face_detection, bgr_image) + (h, w) = bgr_image.shape[:2] + + # for face_coordinates in faces: + for i in range(0, faces.shape[2]): + confidence = faces[0, 0, i, 2] + if confidence < 0.5: + continue + + box = faces[0, 0, i, 3:7] * np.array([w, h, w, h]) + (startX, startY, endX, endY) = box.astype("int") + x_off, y_off = emotion_offsets + x1 = startX - x_off + x2 = endX + x_off + y1 = startY - y_off + y2 = endY + y_off + + face_coordinates=np.array([startX, startY, endX-startX, endY-startY]) + + # x1, x2, y1, y2 = apply_offsets(face_coordinates, emotion_offsets) + gray_face = gray_image[y1:y2, x1:x2] + try: + gray_face = cv2.resize(gray_face, (emotion_target_size)) + except: + continue + + gray_face = preprocess_input(gray_face, True) + gray_face = np.expand_dims(gray_face, 0) + gray_face = np.expand_dims(gray_face, -1) + emotion_prediction = emotion_classifier.predict(gray_face) + emotion_probability = np.max(emotion_prediction) + emotion_label_arg = np.argmax(emotion_prediction) + emotion_text = emotion_labels[emotion_label_arg] + emotion_window.append(emotion_text) + + if len(emotion_window) > frame_window: + emotion_window.pop(0) + try: + emotion_mode = mode(emotion_window) + except: + continue + + if emotion_text == 'angry': + color = emotion_probability * np.asarray((255, 0, 0)) + elif emotion_text == 'sad': + color = emotion_probability * np.asarray((0, 0, 255)) + elif emotion_text == 'happy': + color = emotion_probability * np.asarray((255, 255, 0)) + elif emotion_text == 'surprise': + color = emotion_probability * np.asarray((0, 255, 255)) + else: + color = emotion_probability * np.asarray((0, 255, 0)) + + color = color.astype(int) + color = color.tolist() + + draw_bounding_box(face_coordinates, rgb_image, color) + draw_text(face_coordinates, rgb_image, emotion_mode, + color, 0, -45, 1, 1) + + bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR) + cv2.imshow('window_frame', bgr_image) + if cv2.waitKey(1) & 0xFF == ord('q'): + break +video_capture.release() +cv2.destroyAllWindows()