In Masterarbeit:"Anomalie-Detektion in Zellbildern zur Anwendung der Leukämieerkennung" verwendete CSI Methode.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

eval-checkpoint.ipynb 214KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691
  1. {
  2. "cells": [
  3. {
  4. "cell_type": "markdown",
  5. "id": "5dba5871",
  6. "metadata": {},
  7. "source": [
  8. "# In-Distribution = hem"
  9. ]
  10. },
  11. {
  12. "cell_type": "markdown",
  13. "id": "de7c031b",
  14. "metadata": {},
  15. "source": [
  16. "## combined"
  17. ]
  18. },
  19. {
  20. "cell_type": "code",
  21. "execution_count": 129,
  22. "id": "7c21eb48",
  23. "metadata": {
  24. "scrolled": true
  25. },
  26. "outputs": [
  27. {
  28. "name": "stdout",
  29. "output_type": "stream",
  30. "text": [
  31. "Pre-compute global statistics...\n",
  32. "axis size: 3581 3581 3581 3581\n",
  33. "weight_sim:\t0.0152\t0.0130\t0.0105\t0.0135\n",
  34. "weight_shi:\t-0.0578\t0.1014\t0.1178\t0.1034\n",
  35. "Pre-compute features...\n",
  36. "Compute OOD scores... (score: CSI)\n",
  37. "One_class_real_mean: 0.5853569764733287\n",
  38. "CNMC 1.9901 +- 0.1085 q0: 1.7212 q10: 1.8609 q20: 1.9042 q30: 1.9291 q40: 1.9549 q50: 1.9800 q60: 2.0013 q70: 2.0351 q80: 2.0786 q90: 2.1427 q100: 2.3180\n",
  39. "one_class_0 1.9561 +- 0.0829 q0: 1.6679 q10: 1.8555 q20: 1.8869 q30: 1.9128 q40: 1.9357 q50: 1.9564 q60: 1.9747 q70: 1.9978 q80: 2.0224 q90: 2.0600 q100: 2.2041\n",
  40. "[one_class_0 CSI 0.5854] [one_class_0 best 0.5854] \n",
  41. "[one_class_mean CSI 0.5854] [one_class_mean best 0.5854] \n",
  42. "0.5854\t0.5854\n"
  43. ]
  44. }
  45. ],
  46. "source": [
  47. "# EVALUATION\n",
  48. "# dataset : CNMC\n",
  49. "# res : 450px\n",
  50. "# id_class : hem\n",
  51. "# epoch : 100\n",
  52. "# shift_tr : blur_randpers\n",
  53. "# crop : 0.08\n",
  54. "# blur_sigma : 40\n",
  55. "# randpers : 0.8\n",
  56. "# color_dist : 0.5\n",
  57. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --distortion_scale 0.8 --resize_factor 0.08 --blur_sigma 40 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur_randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_randpers_resize_factor0.08_color_dist0.5_one_class_1/last.model\""
  58. ]
  59. },
  60. {
  61. "cell_type": "code",
  62. "execution_count": 130,
  63. "id": "846efb49",
  64. "metadata": {
  65. "scrolled": true
  66. },
  67. "outputs": [
  68. {
  69. "name": "stdout",
  70. "output_type": "stream",
  71. "text": [
  72. "Pre-compute global statistics...\n",
  73. "axis size: 3581 3581 3581 3581\n",
  74. "weight_sim:\t0.0109\t0.0072\t0.0133\t0.0129\n",
  75. "weight_shi:\t0.4840\t0.0844\t0.4048\t0.2004\n",
  76. "Pre-compute features...\n",
  77. "Compute OOD scores... (score: CSI)\n",
  78. "One_class_real_mean: 0.4842849583244716\n",
  79. "CNMC 1.9963 +- 0.4334 q0: 0.3449 q10: 1.4686 q20: 1.6647 q30: 1.7749 q40: 1.8802 q50: 1.9851 q60: 2.0904 q70: 2.2032 q80: 2.3314 q90: 2.5160 q100: 3.5596\n",
  80. "one_class_0 2.0168 +- 0.3659 q0: 0.5032 q10: 1.5638 q20: 1.7269 q30: 1.8222 q40: 1.9245 q50: 2.0083 q60: 2.0883 q70: 2.1776 q80: 2.3057 q90: 2.4967 q100: 3.3674\n",
  81. "[one_class_0 CSI 0.4843] [one_class_0 best 0.4843] \n",
  82. "[one_class_mean CSI 0.4843] [one_class_mean best 0.4843] \n",
  83. "0.4843\t0.4843\n"
  84. ]
  85. }
  86. ],
  87. "source": [
  88. "# EVALUATION\n",
  89. "# dataset : CNMC\n",
  90. "# res : 450px\n",
  91. "# id_class : hem\n",
  92. "# epoch : 100\n",
  93. "# shift_tr : blur_sharp\n",
  94. "# crop : 0.08\n",
  95. "# blur_sigma : 40\n",
  96. "# randpers : 0.8\n",
  97. "# color_dist : 0.5\n",
  98. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --sharpness_factor 128 --resize_factor 0.08 --blur_sigma 40 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur_sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_sharp_resize_factor0.08_color_dist0.5_one_class_1/last.model\""
  99. ]
  100. },
  101. {
  102. "cell_type": "code",
  103. "execution_count": 131,
  104. "id": "ebf2e296",
  105. "metadata": {
  106. "scrolled": true
  107. },
  108. "outputs": [
  109. {
  110. "name": "stdout",
  111. "output_type": "stream",
  112. "text": [
  113. "Pre-compute global statistics...\n",
  114. "axis size: 3581 3581 3581 3581\n",
  115. "weight_sim:\t0.0019\t0.0039\t0.0042\t0.0047\n",
  116. "weight_shi:\t0.0159\t0.3020\t1.0707\t0.5438\n",
  117. "Pre-compute features...\n",
  118. "Compute OOD scores... (score: CSI)\n",
  119. "One_class_real_mean: 0.43598274238142987\n",
  120. "CNMC 1.9968 +- 0.4243 q0: 1.0210 q10: 1.5387 q20: 1.6392 q30: 1.7221 q40: 1.7914 q50: 1.8964 q60: 2.0368 q70: 2.1923 q80: 2.3638 q90: 2.6239 q100: 3.6290\n",
  121. "one_class_0 2.0836 +- 0.4325 q0: 1.0885 q10: 1.6040 q20: 1.7218 q30: 1.8127 q40: 1.9018 q50: 1.9885 q60: 2.1022 q70: 2.2500 q80: 2.4798 q90: 2.6977 q100: 3.7788\n",
  122. "[one_class_0 CSI 0.4360] [one_class_0 best 0.4360] \n",
  123. "[one_class_mean CSI 0.4360] [one_class_mean best 0.4360] \n",
  124. "0.4360\t0.4360\n"
  125. ]
  126. }
  127. ],
  128. "source": [
  129. "# EVALUATION\n",
  130. "# dataset : CNMC\n",
  131. "# res : 450px\n",
  132. "# id_class : hem\n",
  133. "# epoch : 100\n",
  134. "# shift_tr : randpers_sharp\n",
  135. "# crop : 0.08\n",
  136. "# blur_sigma : 40\n",
  137. "# randpers : 0.8\n",
  138. "# color_dist : 0.5\n",
  139. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --distortion_scale 0.8 --resize_factor 0.08 --sharpness_factor 128 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers_sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_sharp_resize_factor0.08_color_dist0.5_one_class_1/last.model\""
  140. ]
  141. },
  142. {
  143. "cell_type": "code",
  144. "execution_count": 132,
  145. "id": "a7b553d3",
  146. "metadata": {
  147. "scrolled": true
  148. },
  149. "outputs": [
  150. {
  151. "name": "stdout",
  152. "output_type": "stream",
  153. "text": [
  154. "Pre-compute global statistics...\n",
  155. "axis size: 3581 3581 3581 3581\n",
  156. "weight_sim:\t0.0011\t0.0008\t0.0009\t0.0009\n",
  157. "weight_shi:\t-0.0836\t0.1015\t0.0813\t0.0787\n",
  158. "Pre-compute features...\n",
  159. "Compute OOD scores... (score: CSI)\n",
  160. "One_class_real_mean: 0.5724992151024417\n",
  161. "CNMC 2.0160 +- 0.0836 q0: 1.8554 q10: 1.9224 q20: 1.9466 q30: 1.9663 q40: 1.9854 q50: 2.0042 q60: 2.0232 q70: 2.0465 q80: 2.0759 q90: 2.1259 q100: 2.3440\n",
  162. "one_class_0 1.9930 +- 0.0670 q0: 1.8047 q10: 1.9141 q20: 1.9399 q30: 1.9557 q40: 1.9704 q50: 1.9843 q60: 2.0012 q70: 2.0244 q80: 2.0475 q90: 2.0793 q100: 2.2942\n",
  163. "[one_class_0 CSI 0.5725] [one_class_0 best 0.5725] \n",
  164. "[one_class_mean CSI 0.5725] [one_class_mean best 0.5725] \n",
  165. "0.5725\t0.5725\n"
  166. ]
  167. }
  168. ],
  169. "source": [
  170. "# EVALUATION\n",
  171. "# dataset : CNMC\n",
  172. "# res : 450px\n",
  173. "# id_class : hem\n",
  174. "# epoch : 100\n",
  175. "# shift_tr : blur_randpers_sharp\n",
  176. "# crop : 0.08\n",
  177. "# blur_sigma : 40\n",
  178. "# randpers : 0.8\n",
  179. "# color_dist : 0.5\n",
  180. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --sharpness_factor 128 --distortion_scale 0.8 --resize_factor 0.08 --blur_sigma 40 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur_randpers_sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_randpers_sharp_resize_factor0.08_color_dist0.5_one_class_1/last.model\""
  181. ]
  182. },
  183. {
  184. "cell_type": "markdown",
  185. "id": "b5d5f05f",
  186. "metadata": {},
  187. "source": [
  188. "## sharp"
  189. ]
  190. },
  191. {
  192. "cell_type": "code",
  193. "execution_count": 17,
  194. "id": "13c15d92",
  195. "metadata": {},
  196. "outputs": [
  197. {
  198. "name": "stdout",
  199. "output_type": "stream",
  200. "text": [
  201. "Pre-compute global statistics...\n",
  202. "axis size: 3581 3581 3581 3581\n",
  203. "weight_sim:\t0.0082\t0.0048\t0.0035\t0.0035\n",
  204. "weight_shi:\t-0.0162\t0.0291\t0.0264\t0.0261\n",
  205. "Pre-compute features...\n",
  206. "Compute OOD scores... (score: CSI)\n",
  207. "One_class_real_mean: 0.46516067612594825\n",
  208. "CNMC 2.0611 +- 0.2843 q0: 1.4233 q10: 1.7048 q20: 1.8158 q30: 1.8910 q40: 1.9831 q50: 2.0498 q60: 2.1209 q70: 2.1990 q80: 2.3022 q90: 2.4508 q100: 3.0255\n",
  209. "one_class_0 2.0896 +- 0.2109 q0: 1.5218 q10: 1.8407 q20: 1.9143 q30: 1.9720 q40: 2.0252 q50: 2.0691 q60: 2.1174 q70: 2.1761 q80: 2.2568 q90: 2.3717 q100: 2.9418\n",
  210. "[one_class_0 CSI 0.4652] [one_class_0 best 0.4652] \n",
  211. "[one_class_mean CSI 0.4652] [one_class_mean best 0.4652] \n",
  212. "0.4652\t0.4652\n"
  213. ]
  214. }
  215. ],
  216. "source": [
  217. "# EVALUATION\n",
  218. "# dataset : CNMC\n",
  219. "# res : 450px\n",
  220. "# id_class : hem\n",
  221. "# epoch : 100\n",
  222. "# shift_tr : sharp\n",
  223. "# crop : 0.08\n",
  224. "# sharpness : 4096\n",
  225. "# color_dist : 0.5\n",
  226. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 4096 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor4096.0_one_class_1/last.model\""
  227. ]
  228. },
  229. {
  230. "cell_type": "code",
  231. "execution_count": 18,
  232. "id": "25951e79",
  233. "metadata": {},
  234. "outputs": [
  235. {
  236. "name": "stdout",
  237. "output_type": "stream",
  238. "text": [
  239. "Pre-compute global statistics...\n",
  240. "axis size: 3581 3581 3581 3581\n",
  241. "weight_sim:\t0.0095\t0.0075\t0.0068\t0.0072\n",
  242. "weight_shi:\t-0.0480\t0.0769\t0.0704\t0.0693\n",
  243. "Pre-compute features...\n",
  244. "Compute OOD scores... (score: CSI)\n",
  245. "One_class_real_mean: 0.4025752235692077\n",
  246. "CNMC 1.9780 +- 0.1552 q0: 1.6133 q10: 1.7905 q20: 1.8304 q30: 1.8774 q40: 1.9265 q50: 1.9698 q60: 2.0166 q70: 2.0610 q80: 2.1123 q90: 2.1776 q100: 2.5595\n",
  247. "one_class_0 2.0255 +- 0.1272 q0: 1.6800 q10: 1.8659 q20: 1.9179 q30: 1.9585 q40: 1.9884 q50: 2.0210 q60: 2.0530 q70: 2.0845 q80: 2.1202 q90: 2.1844 q100: 2.7354\n",
  248. "[one_class_0 CSI 0.4026] [one_class_0 best 0.4026] \n",
  249. "[one_class_mean CSI 0.4026] [one_class_mean best 0.4026] \n",
  250. "0.4026\t0.4026\n"
  251. ]
  252. }
  253. ],
  254. "source": [
  255. "# EVALUATION\n",
  256. "# dataset : CNMC\n",
  257. "# res : 450px\n",
  258. "# id_class : hem\n",
  259. "# epoch : 100\n",
  260. "# shift_tr : sharp\n",
  261. "# crop : 0.08\n",
  262. "# sharpness : 2048\n",
  263. "# color_dist : 0.5\n",
  264. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 2048 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor2048.0_one_class_1/last.model\""
  265. ]
  266. },
  267. {
  268. "cell_type": "code",
  269. "execution_count": 133,
  270. "id": "4fc12b02",
  271. "metadata": {},
  272. "outputs": [
  273. {
  274. "name": "stdout",
  275. "output_type": "stream",
  276. "text": [
  277. "Pre-compute global statistics...\n",
  278. "axis size: 3581 3581 3581 3581\n",
  279. "weight_sim:\t0.0089\t0.0059\t0.0064\t0.0063\n",
  280. "weight_shi:\t-0.0361\t0.0765\t0.0742\t0.0727\n",
  281. "Pre-compute features...\n",
  282. "Compute OOD scores... (score: CSI)\n",
  283. "One_class_real_mean: 0.4227749420188578\n",
  284. "CNMC 2.0207 +- 0.1832 q0: 1.5627 q10: 1.7726 q20: 1.8618 q30: 1.9226 q40: 1.9811 q50: 2.0217 q60: 2.0627 q70: 2.1123 q80: 2.1832 q90: 2.2560 q100: 2.6861\n",
  285. "one_class_0 2.0632 +- 0.1200 q0: 1.6644 q10: 1.9104 q20: 1.9645 q30: 2.0071 q40: 2.0386 q50: 2.0633 q60: 2.0887 q70: 2.1181 q80: 2.1534 q90: 2.2160 q100: 2.5742\n",
  286. "[one_class_0 CSI 0.4228] [one_class_0 best 0.4228] \n",
  287. "[one_class_mean CSI 0.4228] [one_class_mean best 0.4228] \n",
  288. "0.4228\t0.4228\n"
  289. ]
  290. }
  291. ],
  292. "source": [
  293. "# EVALUATION\n",
  294. "# dataset : CNMC\n",
  295. "# res : 450px\n",
  296. "# id_class : hem\n",
  297. "# epoch : 100\n",
  298. "# shift_tr : sharp\n",
  299. "# crop : 0.08\n",
  300. "# sharpness : 1024\n",
  301. "# color_dist : 0.5\n",
  302. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 1024 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor1024.0_one_class_1/last.model\""
  303. ]
  304. },
  305. {
  306. "cell_type": "code",
  307. "execution_count": 20,
  308. "id": "99698eb6",
  309. "metadata": {
  310. "scrolled": true
  311. },
  312. "outputs": [
  313. {
  314. "name": "stdout",
  315. "output_type": "stream",
  316. "text": [
  317. "Pre-compute global statistics...\n",
  318. "axis size: 3581 3581 3581 3581\n",
  319. "weight_sim:\t0.0067\t0.0032\t0.0035\t0.0038\n",
  320. "weight_shi:\t-0.0499\t0.0682\t0.0675\t0.0722\n",
  321. "Pre-compute features...\n",
  322. "Compute OOD scores... (score: CSI)\n",
  323. "One_class_real_mean: 0.3898554522529092\n",
  324. "CNMC 1.9663 +- 0.1069 q0: 1.6970 q10: 1.8437 q20: 1.8799 q30: 1.9043 q40: 1.9235 q50: 1.9537 q60: 1.9831 q70: 2.0145 q80: 2.0505 q90: 2.1087 q100: 2.5004\n",
  325. "one_class_0 2.0038 +- 0.0970 q0: 1.7568 q10: 1.8902 q20: 1.9239 q30: 1.9445 q40: 1.9657 q50: 1.9897 q60: 2.0196 q70: 2.0483 q80: 2.0868 q90: 2.1398 q100: 2.3773\n",
  326. "[one_class_0 CSI 0.3899] [one_class_0 best 0.3899] \n",
  327. "[one_class_mean CSI 0.3899] [one_class_mean best 0.3899] \n",
  328. "0.3899\t0.3899\n"
  329. ]
  330. }
  331. ],
  332. "source": [
  333. "# EVALUATION\n",
  334. "# dataset : CNMC\n",
  335. "# res : 450px\n",
  336. "# id_class : hem\n",
  337. "# epoch : 100\n",
  338. "# shift_tr : sharp\n",
  339. "# crop : 0.08\n",
  340. "# sharpness : 512\n",
  341. "# color_dist : 0.5\n",
  342. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 512 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor512.0_one_class_1/last.model\""
  343. ]
  344. },
  345. {
  346. "cell_type": "code",
  347. "execution_count": 21,
  348. "id": "01e6d61a",
  349. "metadata": {},
  350. "outputs": [
  351. {
  352. "name": "stdout",
  353. "output_type": "stream",
  354. "text": [
  355. "Pre-compute global statistics...\n",
  356. "axis size: 3581 3581 3581 3581\n",
  357. "weight_sim:\t0.0053\t0.0084\t0.0092\t0.0087\n",
  358. "weight_shi:\t0.4300\t0.0647\t0.0695\t0.0685\n",
  359. "Pre-compute features...\n",
  360. "Compute OOD scores... (score: CSI)\n",
  361. "One_class_real_mean: 0.5784846919656873\n",
  362. "CNMC 2.1270 +- 0.9610 q0: -0.9326 q10: 0.9101 q20: 1.3880 q30: 1.6010 q40: 1.8639 q50: 2.1067 q60: 2.3374 q70: 2.5413 q80: 2.8893 q90: 3.3775 q100: 5.1585\n",
  363. "one_class_0 1.8950 +- 0.7309 q0: -0.2104 q10: 1.0100 q20: 1.3020 q30: 1.4936 q40: 1.6684 q50: 1.8373 q60: 2.0139 q70: 2.2017 q80: 2.4870 q90: 2.8570 q100: 4.5441\n",
  364. "[one_class_0 CSI 0.5785] [one_class_0 best 0.5785] \n",
  365. "[one_class_mean CSI 0.5785] [one_class_mean best 0.5785] \n",
  366. "0.5785\t0.5785\n"
  367. ]
  368. }
  369. ],
  370. "source": [
  371. "# EVALUATION\n",
  372. "# dataset : CNMC\n",
  373. "# res : 450px\n",
  374. "# id_class : hem\n",
  375. "# epoch : 100\n",
  376. "# shift_tr : sharp\n",
  377. "# crop : 0.08\n",
  378. "# sharpness : 256\n",
  379. "# color_dist : 0.5\n",
  380. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 256 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor256.0_one_class_1/last.model\""
  381. ]
  382. },
  383. {
  384. "cell_type": "code",
  385. "execution_count": 22,
  386. "id": "65cc4fcd",
  387. "metadata": {},
  388. "outputs": [
  389. {
  390. "name": "stdout",
  391. "output_type": "stream",
  392. "text": [
  393. "Pre-compute global statistics...\n",
  394. "axis size: 3581 3581 3581 3581\n",
  395. "weight_sim:\t0.0089\t0.0063\t0.0075\t0.0065\n",
  396. "weight_shi:\t-0.0184\t0.0363\t0.0371\t0.0371\n",
  397. "Pre-compute features...\n",
  398. "Compute OOD scores... (score: CSI)\n",
  399. "One_class_real_mean: 0.3679688370350115\n",
  400. "CNMC 1.9800 +- 0.0919 q0: 1.7207 q10: 1.8629 q20: 1.8911 q30: 1.9241 q40: 1.9548 q50: 1.9755 q60: 2.0034 q70: 2.0354 q80: 2.0631 q90: 2.1071 q100: 2.2242\n",
  401. "one_class_0 2.0217 +- 0.0794 q0: 1.7727 q10: 1.9194 q20: 1.9543 q30: 1.9779 q40: 1.9999 q50: 2.0212 q60: 2.0423 q70: 2.0650 q80: 2.0906 q90: 2.1259 q100: 2.2548\n",
  402. "[one_class_0 CSI 0.3680] [one_class_0 best 0.3680] \n",
  403. "[one_class_mean CSI 0.3680] [one_class_mean best 0.3680] \n",
  404. "0.3680\t0.3680\n"
  405. ]
  406. }
  407. ],
  408. "source": [
  409. "# EVALUATION\n",
  410. "# dataset : CNMC\n",
  411. "# res : 450px\n",
  412. "# id_class : hem\n",
  413. "# epoch : 100\n",
  414. "# shift_tr : sharp\n",
  415. "# crop : 0.08\n",
  416. "# sharpness : 150\n",
  417. "# color_dist : 0.5\n",
  418. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 150 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor150.0_one_class_1/last.model\""
  419. ]
  420. },
  421. {
  422. "cell_type": "code",
  423. "execution_count": 23,
  424. "id": "e13b48db",
  425. "metadata": {},
  426. "outputs": [
  427. {
  428. "name": "stdout",
  429. "output_type": "stream",
  430. "text": [
  431. "Pre-compute global statistics...\n",
  432. "axis size: 3581 3581 3581 3581\n",
  433. "weight_sim:\t0.0083\t0.0053\t0.0056\t0.0053\n",
  434. "weight_shi:\t-0.1256\t0.0869\t0.0823\t0.0921\n",
  435. "Pre-compute features...\n",
  436. "Compute OOD scores... (score: CSI)\n",
  437. "One_class_real_mean: 0.4935733347512128\n",
  438. "CNMC 2.0389 +- 0.1184 q0: 1.7300 q10: 1.8883 q20: 1.9358 q30: 1.9771 q40: 2.0079 q50: 2.0342 q60: 2.0657 q70: 2.0974 q80: 2.1469 q90: 2.1945 q100: 2.5086\n",
  439. "one_class_0 2.0418 +- 0.0930 q0: 1.7624 q10: 1.9334 q20: 1.9610 q30: 1.9867 q40: 2.0125 q50: 2.0354 q60: 2.0608 q70: 2.0915 q80: 2.1163 q90: 2.1599 q100: 2.3964\n",
  440. "[one_class_0 CSI 0.4936] [one_class_0 best 0.4936] \n",
  441. "[one_class_mean CSI 0.4936] [one_class_mean best 0.4936] \n",
  442. "0.4936\t0.4936\n"
  443. ]
  444. }
  445. ],
  446. "source": [
  447. "# EVALUATION\n",
  448. "# dataset : CNMC\n",
  449. "# res : 450px\n",
  450. "# id_class : hem\n",
  451. "# epoch : 100\n",
  452. "# shift_tr : sharp\n",
  453. "# crop : 0.08\n",
  454. "# sharpness : 140\n",
  455. "# color_dist : 0.5\n",
  456. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 140 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor140.0_one_class_1/last.model\""
  457. ]
  458. },
  459. {
  460. "cell_type": "code",
  461. "execution_count": 24,
  462. "id": "29cf690f",
  463. "metadata": {},
  464. "outputs": [
  465. {
  466. "name": "stdout",
  467. "output_type": "stream",
  468. "text": [
  469. "Pre-compute global statistics...\n",
  470. "axis size: 3581 3581 3581 3581\n",
  471. "weight_sim:\t0.0045\t0.0043\t0.0070\t0.0053\n",
  472. "weight_shi:\t-0.0813\t0.0676\t0.0710\t0.0626\n",
  473. "Pre-compute features...\n",
  474. "Compute OOD scores... (score: CSI)\n",
  475. "One_class_real_mean: 0.4419042880725954\n",
  476. "CNMC 2.0129 +- 0.1195 q0: 1.5835 q10: 1.8643 q20: 1.9252 q30: 1.9581 q40: 1.9900 q50: 2.0157 q60: 2.0378 q70: 2.0653 q80: 2.1055 q90: 2.1542 q100: 2.4828\n",
  477. "one_class_0 2.0362 +- 0.1010 q0: 1.6891 q10: 1.9098 q20: 1.9486 q30: 1.9817 q40: 2.0087 q50: 2.0351 q60: 2.0576 q70: 2.0853 q80: 2.1206 q90: 2.1651 q100: 2.3927\n",
  478. "[one_class_0 CSI 0.4419] [one_class_0 best 0.4419] \n",
  479. "[one_class_mean CSI 0.4419] [one_class_mean best 0.4419] \n",
  480. "0.4419\t0.4419\n"
  481. ]
  482. }
  483. ],
  484. "source": [
  485. "# EVALUATION\n",
  486. "# dataset : CNMC\n",
  487. "# res : 450px\n",
  488. "# id_class : hem\n",
  489. "# epoch : 100\n",
  490. "# shift_tr : sharp\n",
  491. "# crop : 0.08\n",
  492. "# sharpness : 130\n",
  493. "# color_dist : 0.5\n",
  494. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 130 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor130.0_one_class_1/last.model\""
  495. ]
  496. },
  497. {
  498. "cell_type": "code",
  499. "execution_count": 25,
  500. "id": "dfaa2119",
  501. "metadata": {},
  502. "outputs": [
  503. {
  504. "name": "stdout",
  505. "output_type": "stream",
  506. "text": [
  507. "Pre-compute global statistics...\n",
  508. "axis size: 3581 3581 3581 3581\n",
  509. "weight_sim:\t0.0130\t0.0032\t0.0032\t0.0028\n",
  510. "weight_shi:\t-0.0796\t0.1288\t0.1192\t0.1286\n",
  511. "Pre-compute features...\n",
  512. "Compute OOD scores... (score: CSI)\n",
  513. "One_class_real_mean: 0.47003337080586194\n",
  514. "CNMC 1.9969 +- 0.1287 q0: 1.5279 q10: 1.8373 q20: 1.9073 q30: 1.9387 q40: 1.9663 q50: 1.9928 q60: 2.0270 q70: 2.0640 q80: 2.0993 q90: 2.1517 q100: 2.4161\n",
  515. "one_class_0 2.0110 +- 0.1133 q0: 1.6407 q10: 1.8709 q20: 1.9143 q30: 1.9536 q40: 1.9857 q50: 2.0121 q60: 2.0374 q70: 2.0675 q80: 2.1043 q90: 2.1554 q100: 2.3778\n",
  516. "[one_class_0 CSI 0.4700] [one_class_0 best 0.4700] \n",
  517. "[one_class_mean CSI 0.4700] [one_class_mean best 0.4700] \n",
  518. "0.4700\t0.4700\n"
  519. ]
  520. }
  521. ],
  522. "source": [
  523. "# EVALUATION\n",
  524. "# dataset : CNMC\n",
  525. "# res : 450px\n",
  526. "# id_class : hem\n",
  527. "# epoch : 100\n",
  528. "# shift_tr : sharp\n",
  529. "# crop : 0.08\n",
  530. "# sharpness : 120\n",
  531. "# color_dist : 0.5\n",
  532. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 120 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor120.0_one_class_1/last.model\""
  533. ]
  534. },
  535. {
  536. "cell_type": "code",
  537. "execution_count": 26,
  538. "id": "e3eecf30",
  539. "metadata": {},
  540. "outputs": [
  541. {
  542. "name": "stdout",
  543. "output_type": "stream",
  544. "text": [
  545. "Pre-compute global statistics...\n",
  546. "axis size: 3581 3581 3581 3581\n",
  547. "weight_sim:\t0.0091\t0.0036\t0.0042\t0.0040\n",
  548. "weight_shi:\t0.2410\t0.5432\t0.2487\t0.3103\n",
  549. "Pre-compute features...\n",
  550. "Compute OOD scores... (score: CSI)\n",
  551. "One_class_real_mean: 0.5940902277722075\n",
  552. "CNMC 2.0988 +- 0.5314 q0: 1.0497 q10: 1.4954 q20: 1.6747 q30: 1.8162 q40: 1.9078 q50: 2.0314 q60: 2.1299 q70: 2.2698 q80: 2.4594 q90: 2.8458 q100: 4.3420\n",
  553. "one_class_0 1.9324 +- 0.3714 q0: 1.0642 q10: 1.5287 q20: 1.6460 q30: 1.7249 q40: 1.8095 q50: 1.8731 q60: 1.9535 q70: 2.0458 q80: 2.1797 q90: 2.4138 q100: 3.6869\n",
  554. "[one_class_0 CSI 0.5941] [one_class_0 best 0.5941] \n",
  555. "[one_class_mean CSI 0.5941] [one_class_mean best 0.5941] \n",
  556. "0.5941\t0.5941\n"
  557. ]
  558. }
  559. ],
  560. "source": [
  561. "# EVALUATION\n",
  562. "# dataset : CNMC\n",
  563. "# res : 450px\n",
  564. "# id_class : hem\n",
  565. "# epoch : 100\n",
  566. "# shift_tr : sharp\n",
  567. "# crop : 0.08\n",
  568. "# sharpness : 128\n",
  569. "# color_dist : 0.5\n",
  570. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 128 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor128.0_one_class_1/last.model\""
  571. ]
  572. },
  573. {
  574. "cell_type": "code",
  575. "execution_count": 27,
  576. "id": "d7d86bff",
  577. "metadata": {},
  578. "outputs": [
  579. {
  580. "name": "stdout",
  581. "output_type": "stream",
  582. "text": [
  583. "Pre-compute global statistics...\n",
  584. "axis size: 3581 3581 3581 3581\n",
  585. "weight_sim:\t0.0077\t0.0039\t0.0057\t0.0045\n",
  586. "weight_shi:\t-0.0543\t0.1223\t0.1116\t0.1079\n",
  587. "Pre-compute features...\n",
  588. "Compute OOD scores... (score: CSI)\n",
  589. "One_class_real_mean: 0.4230414273995078\n",
  590. "CNMC 1.9898 +- 0.1931 q0: 1.3874 q10: 1.7331 q20: 1.8325 q30: 1.9150 q40: 1.9682 q50: 2.0080 q60: 2.0511 q70: 2.1020 q80: 2.1509 q90: 2.2176 q100: 2.4975\n",
  591. "one_class_0 2.0442 +- 0.1594 q0: 1.5034 q10: 1.8378 q20: 1.9120 q30: 1.9673 q40: 2.0118 q50: 2.0508 q60: 2.0920 q70: 2.1314 q80: 2.1747 q90: 2.2473 q100: 2.5530\n",
  592. "[one_class_0 CSI 0.4230] [one_class_0 best 0.4230] \n",
  593. "[one_class_mean CSI 0.4230] [one_class_mean best 0.4230] \n",
  594. "0.4230\t0.4230\n"
  595. ]
  596. }
  597. ],
  598. "source": [
  599. "# EVALUATION\n",
  600. "# dataset : CNMC\n",
  601. "# res : 450px\n",
  602. "# id_class : hem\n",
  603. "# epoch : 100\n",
  604. "# shift_tr : sharp\n",
  605. "# crop : 0.08\n",
  606. "# sharpness : 100\n",
  607. "# color_dist : 0.5\n",
  608. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 100 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor100.0_one_class_1/last.model\""
  609. ]
  610. },
  611. {
  612. "cell_type": "code",
  613. "execution_count": 28,
  614. "id": "d60476b1",
  615. "metadata": {},
  616. "outputs": [
  617. {
  618. "name": "stdout",
  619. "output_type": "stream",
  620. "text": [
  621. "Pre-compute global statistics...\n",
  622. "axis size: 3581 3581 3581 3581\n",
  623. "weight_sim:\t0.0071\t0.0035\t0.0055\t0.0033\n",
  624. "weight_shi:\t-0.7731\t-0.4426\t3.0750\t-1.0296\n",
  625. "Pre-compute features...\n",
  626. "Compute OOD scores... (score: CSI)\n",
  627. "One_class_real_mean: 0.3861885880958892\n",
  628. "CNMC 1.8504 +- 1.7281 q0: -3.9915 q10: -0.3957 q20: 0.3727 q30: 1.0483 q40: 1.6967 q50: 2.0760 q60: 2.4763 q70: 2.8382 q80: 3.3079 q90: 3.8465 q100: 5.6520\n",
  629. "one_class_0 2.5429 +- 1.3399 q0: -4.5019 q10: 0.9296 q20: 1.4679 q30: 1.9042 q40: 2.2539 q50: 2.5979 q60: 2.9289 q70: 3.3053 q80: 3.6585 q90: 4.1959 q100: 6.6848\n",
  630. "[one_class_0 CSI 0.3862] [one_class_0 best 0.3862] \n",
  631. "[one_class_mean CSI 0.3862] [one_class_mean best 0.3862] \n",
  632. "0.3862\t0.3862\n"
  633. ]
  634. }
  635. ],
  636. "source": [
  637. "# EVALUATION\n",
  638. "# dataset : CNMC\n",
  639. "# res : 450px\n",
  640. "# id_class : hem\n",
  641. "# epoch : 100\n",
  642. "# shift_tr : sharp\n",
  643. "# crop : 0.08\n",
  644. "# sharpness : 80\n",
  645. "# color_dist : 0.5\n",
  646. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 80 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor80.0_one_class_1/last.model\""
  647. ]
  648. },
  649. {
  650. "cell_type": "code",
  651. "execution_count": 29,
  652. "id": "b367669a",
  653. "metadata": {},
  654. "outputs": [
  655. {
  656. "name": "stdout",
  657. "output_type": "stream",
  658. "text": [
  659. "Pre-compute global statistics...\n",
  660. "axis size: 3581 3581 3581 3581\n",
  661. "weight_sim:\t0.0083\t0.0112\t0.0076\t0.0136\n",
  662. "weight_shi:\t-0.0567\t0.1140\t0.0842\t0.1028\n",
  663. "Pre-compute features...\n",
  664. "Compute OOD scores... (score: CSI)\n",
  665. "One_class_real_mean: 0.376367240907848\n",
  666. "CNMC 1.9968 +- 0.0768 q0: 1.7939 q10: 1.9005 q20: 1.9341 q30: 1.9569 q40: 1.9751 q50: 1.9937 q60: 2.0157 q70: 2.0367 q80: 2.0629 q90: 2.0964 q100: 2.2761\n",
  667. "one_class_0 2.0289 +- 0.0677 q0: 1.8223 q10: 1.9439 q20: 1.9701 q30: 1.9928 q40: 2.0111 q50: 2.0279 q60: 2.0448 q70: 2.0625 q80: 2.0815 q90: 2.1167 q100: 2.3343\n",
  668. "[one_class_0 CSI 0.3764] [one_class_0 best 0.3764] \n",
  669. "[one_class_mean CSI 0.3764] [one_class_mean best 0.3764] \n",
  670. "0.3764\t0.3764\n"
  671. ]
  672. }
  673. ],
  674. "source": [
  675. "# EVALUATION\n",
  676. "# dataset : CNMC\n",
  677. "# res : 450px\n",
  678. "# id_class : hem\n",
  679. "# epoch : 100\n",
  680. "# shift_tr : sharp\n",
  681. "# crop : 0.08\n",
  682. "# sharpness : 64\n",
  683. "# color_dist : 0.5\n",
  684. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 64 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor64.0_one_class_1/last.model\""
  685. ]
  686. },
  687. {
  688. "cell_type": "code",
  689. "execution_count": 30,
  690. "id": "dce638a8",
  691. "metadata": {},
  692. "outputs": [
  693. {
  694. "name": "stdout",
  695. "output_type": "stream",
  696. "text": [
  697. "Pre-compute global statistics...\n",
  698. "axis size: 3581 3581 3581 3581\n",
  699. "weight_sim:\t0.0150\t0.0058\t0.0129\t0.0054\n",
  700. "weight_shi:\t-0.0982\t0.1165\t0.1059\t0.0929\n",
  701. "Pre-compute features...\n",
  702. "Compute OOD scores... (score: CSI)\n",
  703. "One_class_real_mean: 0.4102051874132815\n",
  704. "CNMC 1.9836 +- 0.2025 q0: 1.4858 q10: 1.7023 q20: 1.8012 q30: 1.8770 q40: 1.9428 q50: 2.0013 q60: 2.0495 q70: 2.0936 q80: 2.1462 q90: 2.2425 q100: 2.5144\n",
  705. "one_class_0 2.0499 +- 0.1863 q0: 1.5414 q10: 1.8119 q20: 1.8950 q30: 1.9488 q40: 1.9984 q50: 2.0487 q60: 2.0962 q70: 2.1512 q80: 2.2168 q90: 2.2846 q100: 2.6101\n",
  706. "[one_class_0 CSI 0.4102] [one_class_0 best 0.4102] \n",
  707. "[one_class_mean CSI 0.4102] [one_class_mean best 0.4102] \n",
  708. "0.4102\t0.4102\n"
  709. ]
  710. }
  711. ],
  712. "source": [
  713. "# EVALUATION\n",
  714. "# dataset : CNMC\n",
  715. "# res : 450px\n",
  716. "# id_class : hem\n",
  717. "# epoch : 100\n",
  718. "# shift_tr : sharp\n",
  719. "# crop : 0.08\n",
  720. "# sharpness : 32\n",
  721. "# color_dist : 0.5\n",
  722. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 32 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor32.0_one_class_1/last.model\""
  723. ]
  724. },
  725. {
  726. "cell_type": "code",
  727. "execution_count": 31,
  728. "id": "28387a64",
  729. "metadata": {},
  730. "outputs": [
  731. {
  732. "name": "stdout",
  733. "output_type": "stream",
  734. "text": [
  735. "Pre-compute global statistics...\n",
  736. "axis size: 3581 3581 3581 3581\n",
  737. "weight_sim:\t0.0088\t0.0070\t0.0079\t0.0070\n",
  738. "weight_shi:\t-0.0517\t0.1752\t0.1985\t0.2796\n",
  739. "Pre-compute features...\n",
  740. "Compute OOD scores... (score: CSI)\n",
  741. "One_class_real_mean: 0.4567964532758079\n",
  742. "CNMC 2.0252 +- 0.2349 q0: 1.3683 q10: 1.7184 q20: 1.8035 q30: 1.8906 q40: 1.9540 q50: 2.0153 q60: 2.0821 q70: 2.1619 q80: 2.2506 q90: 2.3384 q100: 2.6113\n",
  743. "one_class_0 2.0617 +- 0.2072 q0: 1.5917 q10: 1.7983 q20: 1.8819 q30: 1.9423 q40: 1.9904 q50: 2.0477 q60: 2.1076 q70: 2.1646 q80: 2.2533 q90: 2.3431 q100: 2.6298\n",
  744. "[one_class_0 CSI 0.4568] [one_class_0 best 0.4568] \n",
  745. "[one_class_mean CSI 0.4568] [one_class_mean best 0.4568] \n",
  746. "0.4568\t0.4568\n"
  747. ]
  748. }
  749. ],
  750. "source": [
  751. "# EVALUATION\n",
  752. "# dataset : CNMC\n",
  753. "# res : 450px\n",
  754. "# id_class : hem\n",
  755. "# epoch : 100\n",
  756. "# shift_tr : sharp\n",
  757. "# crop : 0.08\n",
  758. "# sharpness : 16\n",
  759. "# color_dist : 0.5\n",
  760. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 16 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor16.0_one_class_1/last.model\""
  761. ]
  762. },
  763. {
  764. "cell_type": "code",
  765. "execution_count": 32,
  766. "id": "424cd4b8",
  767. "metadata": {},
  768. "outputs": [
  769. {
  770. "name": "stdout",
  771. "output_type": "stream",
  772. "text": [
  773. "Pre-compute global statistics...\n",
  774. "axis size: 3581 3581 3581 3581\n",
  775. "weight_sim:\t0.0167\t0.0112\t0.0119\t0.0098\n",
  776. "weight_shi:\t-0.1065\t0.1467\t0.1401\t0.1203\n",
  777. "Pre-compute features...\n",
  778. "Compute OOD scores... (score: CSI)\n",
  779. "One_class_real_mean: 0.4809307872269316\n",
  780. "CNMC 2.0080 +- 0.2589 q0: 1.4062 q10: 1.6813 q20: 1.7879 q30: 1.8753 q40: 1.9309 q50: 1.9964 q60: 2.0601 q70: 2.1217 q80: 2.2165 q90: 2.3636 q100: 3.0258\n",
  781. "one_class_0 2.0288 +- 0.2475 q0: 1.4597 q10: 1.7282 q20: 1.8162 q30: 1.8799 q40: 1.9422 q50: 1.9987 q60: 2.0671 q70: 2.1423 q80: 2.2336 q90: 2.3568 q100: 3.5008\n",
  782. "[one_class_0 CSI 0.4809] [one_class_0 best 0.4809] \n",
  783. "[one_class_mean CSI 0.4809] [one_class_mean best 0.4809] \n",
  784. "0.4809\t0.4809\n"
  785. ]
  786. }
  787. ],
  788. "source": [
  789. "# EVALUATION\n",
  790. "# dataset : CNMC\n",
  791. "# res : 450px\n",
  792. "# id_class : hem\n",
  793. "# epoch : 100\n",
  794. "# shift_tr : sharp\n",
  795. "# crop : 0.08\n",
  796. "# sharpness : 8\n",
  797. "# color_dist : 0.5\n",
  798. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 8 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor8.0_one_class_1/last.model\""
  799. ]
  800. },
  801. {
  802. "cell_type": "code",
  803. "execution_count": 33,
  804. "id": "b30452ce",
  805. "metadata": {},
  806. "outputs": [
  807. {
  808. "name": "stdout",
  809. "output_type": "stream",
  810. "text": [
  811. "Pre-compute global statistics...\n",
  812. "axis size: 3581 3581 3581 3581\n",
  813. "weight_sim:\t0.0033\t0.0030\t0.0025\t0.0028\n",
  814. "weight_shi:\t-0.0191\t0.0502\t0.0455\t0.0473\n",
  815. "Pre-compute features...\n",
  816. "Compute OOD scores... (score: CSI)\n",
  817. "One_class_real_mean: 0.4414099292073041\n",
  818. "CNMC 1.9717 +- 0.1514 q0: 1.4512 q10: 1.7885 q20: 1.8443 q30: 1.8923 q40: 1.9275 q50: 1.9674 q60: 2.0006 q70: 2.0525 q80: 2.0992 q90: 2.1705 q100: 2.4112\n",
  819. "one_class_0 1.9985 +- 0.1198 q0: 1.6117 q10: 1.8478 q20: 1.9013 q30: 1.9366 q40: 1.9671 q50: 1.9967 q60: 2.0257 q70: 2.0616 q80: 2.0994 q90: 2.1570 q100: 2.3699\n",
  820. "[one_class_0 CSI 0.4414] [one_class_0 best 0.4414] \n",
  821. "[one_class_mean CSI 0.4414] [one_class_mean best 0.4414] \n",
  822. "0.4414\t0.4414\n"
  823. ]
  824. }
  825. ],
  826. "source": [
  827. "# EVALUATION\n",
  828. "# dataset : CNMC\n",
  829. "# res : 450px\n",
  830. "# id_class : hem\n",
  831. "# epoch : 100\n",
  832. "# shift_tr : sharp\n",
  833. "# crop : 0.08\n",
  834. "# sharpness : 5\n",
  835. "# color_dist : 0.5\n",
  836. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 5 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor5.0_one_class_1/last.model\""
  837. ]
  838. },
  839. {
  840. "cell_type": "code",
  841. "execution_count": 34,
  842. "id": "61511c88",
  843. "metadata": {},
  844. "outputs": [
  845. {
  846. "name": "stdout",
  847. "output_type": "stream",
  848. "text": [
  849. "Pre-compute global statistics...\n",
  850. "axis size: 3581 3581 3581 3581\n",
  851. "weight_sim:\t0.0048\t0.0048\t0.0033\t0.0040\n",
  852. "weight_shi:\t-0.0216\t0.0573\t0.0466\t0.0474\n",
  853. "Pre-compute features...\n",
  854. "Compute OOD scores... (score: CSI)\n",
  855. "One_class_real_mean: 0.3330541883146477\n",
  856. "CNMC 1.9613 +- 0.1613 q0: 1.4001 q10: 1.7205 q20: 1.8317 q30: 1.8884 q40: 1.9420 q50: 1.9740 q60: 2.0204 q70: 2.0591 q80: 2.1023 q90: 2.1507 q100: 2.3497\n",
  857. "one_class_0 2.0506 +- 0.1298 q0: 1.5709 q10: 1.8729 q20: 1.9474 q30: 2.0023 q40: 2.0388 q50: 2.0668 q60: 2.0970 q70: 2.1272 q80: 2.1610 q90: 2.1981 q100: 2.4918\n",
  858. "[one_class_0 CSI 0.3331] [one_class_0 best 0.3331] \n",
  859. "[one_class_mean CSI 0.3331] [one_class_mean best 0.3331] \n",
  860. "0.3331\t0.3331\n"
  861. ]
  862. }
  863. ],
  864. "source": [
  865. "# EVALUATION\n",
  866. "# dataset : CNMC\n",
  867. "# res : 450px\n",
  868. "# id_class : hem\n",
  869. "# epoch : 100\n",
  870. "# shift_tr : sharp\n",
  871. "# crop : 0.08\n",
  872. "# sharpness : 4\n",
  873. "# color_dist : 0.5\n",
  874. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 4 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor4.0_one_class_1/last.model\""
  875. ]
  876. },
  877. {
  878. "cell_type": "code",
  879. "execution_count": 35,
  880. "id": "9aa87298",
  881. "metadata": {},
  882. "outputs": [
  883. {
  884. "name": "stdout",
  885. "output_type": "stream",
  886. "text": [
  887. "Pre-compute global statistics...\n",
  888. "axis size: 3581 3581 3581 3581\n",
  889. "weight_sim:\t0.0025\t0.0027\t0.0023\t0.0026\n",
  890. "weight_shi:\t-0.0247\t0.0743\t0.0809\t0.0786\n",
  891. "Pre-compute features...\n",
  892. "Compute OOD scores... (score: CSI)\n",
  893. "One_class_real_mean: 0.32097499468295204\n",
  894. "CNMC 1.9756 +- 0.0921 q0: 1.6810 q10: 1.8631 q20: 1.9010 q30: 1.9281 q40: 1.9493 q50: 1.9693 q60: 1.9932 q70: 2.0205 q80: 2.0561 q90: 2.0987 q100: 2.2881\n",
  895. "one_class_0 2.0291 +- 0.0715 q0: 1.8113 q10: 1.9469 q20: 1.9690 q30: 1.9889 q40: 2.0060 q50: 2.0221 q60: 2.0372 q70: 2.0598 q80: 2.0847 q90: 2.1253 q100: 2.3137\n",
  896. "[one_class_0 CSI 0.3210] [one_class_0 best 0.3210] \n",
  897. "[one_class_mean CSI 0.3210] [one_class_mean best 0.3210] \n",
  898. "0.3210\t0.3210\n"
  899. ]
  900. }
  901. ],
  902. "source": [
  903. "# EVALUATION\n",
  904. "# dataset : CNMC\n",
  905. "# res : 450px\n",
  906. "# id_class : hem\n",
  907. "# epoch : 100\n",
  908. "# shift_tr : sharp\n",
  909. "# crop : 0.08\n",
  910. "# sharpness : 3\n",
  911. "# color_dist : 0.5\n",
  912. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 3 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor3.0_one_class_1/last.model\""
  913. ]
  914. },
  915. {
  916. "cell_type": "code",
  917. "execution_count": 36,
  918. "id": "ed261f4c",
  919. "metadata": {},
  920. "outputs": [
  921. {
  922. "name": "stdout",
  923. "output_type": "stream",
  924. "text": [
  925. "Pre-compute global statistics...\n",
  926. "axis size: 3581 3581 3581 3581\n",
  927. "weight_sim:\t0.0016\t0.0017\t0.0018\t0.0018\n",
  928. "weight_shi:\t-0.0191\t0.0634\t0.0692\t0.0681\n",
  929. "Pre-compute features...\n",
  930. "Compute OOD scores... (score: CSI)\n",
  931. "One_class_real_mean: 0.5479179959286604\n",
  932. "CNMC 2.0439 +- 0.2491 q0: 1.3671 q10: 1.7149 q20: 1.8289 q30: 1.9114 q40: 1.9969 q50: 2.0563 q60: 2.1157 q70: 2.1758 q80: 2.2482 q90: 2.3551 q100: 2.6738\n",
  933. "one_class_0 2.0114 +- 0.1906 q0: 1.4681 q10: 1.7626 q20: 1.8467 q30: 1.9083 q40: 1.9608 q50: 2.0043 q60: 2.0576 q70: 2.1069 q80: 2.1710 q90: 2.2558 q100: 2.5480\n",
  934. "[one_class_0 CSI 0.5479] [one_class_0 best 0.5479] \n",
  935. "[one_class_mean CSI 0.5479] [one_class_mean best 0.5479] \n",
  936. "0.5479\t0.5479\n"
  937. ]
  938. }
  939. ],
  940. "source": [
  941. "# EVALUATION\n",
  942. "# dataset : CNMC\n",
  943. "# res : 450px\n",
  944. "# id_class : hem\n",
  945. "# epoch : 100\n",
  946. "# shift_tr : sharp\n",
  947. "# crop : 0.08\n",
  948. "# sharpness : 2\n",
  949. "# color_dist : 0.5\n",
  950. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --sharpness_factor 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor2.0_one_class_1/last.model\""
  951. ]
  952. },
  953. {
  954. "cell_type": "markdown",
  955. "id": "3f347111",
  956. "metadata": {},
  957. "source": [
  958. "## randpers"
  959. ]
  960. },
  961. {
  962. "cell_type": "code",
  963. "execution_count": 37,
  964. "id": "6954e9f3",
  965. "metadata": {},
  966. "outputs": [
  967. {
  968. "name": "stdout",
  969. "output_type": "stream",
  970. "text": [
  971. "Pre-compute global statistics...\n",
  972. "axis size: 3581 3581 3581 3581\n",
  973. "weight_sim:\t0.0027\t0.0028\t0.0028\t0.0029\n",
  974. "weight_shi:\t0.0396\t-0.1267\t-0.1178\t-0.1344\n",
  975. "Pre-compute features...\n",
  976. "Compute OOD scores... (score: CSI)\n",
  977. "One_class_real_mean: 0.35701318627897793\n",
  978. "CNMC 1.9933 +- 0.0426 q0: 1.7637 q10: 1.9470 q20: 1.9683 q30: 1.9825 q40: 1.9896 q50: 1.9981 q60: 2.0059 q70: 2.0119 q80: 2.0228 q90: 2.0391 q100: 2.1039\n",
  979. "one_class_0 2.0107 +- 0.0300 q0: 1.8398 q10: 1.9753 q20: 1.9909 q30: 2.0006 q40: 2.0073 q50: 2.0134 q60: 2.0197 q70: 2.0245 q80: 2.0323 q90: 2.0429 q100: 2.0991\n",
  980. "[one_class_0 CSI 0.3570] [one_class_0 best 0.3570] \n",
  981. "[one_class_mean CSI 0.3570] [one_class_mean best 0.3570] \n",
  982. "0.3570\t0.3570\n"
  983. ]
  984. }
  985. ],
  986. "source": [
  987. "# EVALUATION\n",
  988. "# dataset : CNMC\n",
  989. "# res : 450px\n",
  990. "# id_class : hem\n",
  991. "# epoch : 100\n",
  992. "# shift_tr : randpers\n",
  993. "# crop : 0.08\n",
  994. "# randpers : 0.95\n",
  995. "# color_dist : 0.5\n",
  996. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --distortion_scale 0.95 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.95_one_class_1/last.model\""
  997. ]
  998. },
  999. {
  1000. "cell_type": "code",
  1001. "execution_count": 38,
  1002. "id": "7ef390e9",
  1003. "metadata": {},
  1004. "outputs": [
  1005. {
  1006. "name": "stdout",
  1007. "output_type": "stream",
  1008. "text": [
  1009. "Pre-compute global statistics...\n",
  1010. "axis size: 3581 3581 3581 3581\n",
  1011. "weight_sim:\t0.0079\t0.0098\t0.0115\t0.0104\n",
  1012. "weight_shi:\t-0.2285\t-6.8399\t0.4918\t0.3229\n",
  1013. "Pre-compute features...\n",
  1014. "Compute OOD scores... (score: CSI)\n",
  1015. "One_class_real_mean: 0.5641122049038374\n",
  1016. "CNMC 1.9712 +- 0.8313 q0: -4.4253 q10: 0.9802 q20: 1.5045 q30: 1.8002 q40: 1.9917 q50: 2.1173 q60: 2.2353 q70: 2.3608 q80: 2.5048 q90: 2.7707 q100: 4.2407\n",
  1017. "one_class_0 1.9180 +- 0.6218 q0: -3.1624 q10: 1.2584 q20: 1.5541 q30: 1.7231 q40: 1.8312 q50: 1.9474 q60: 2.0646 q70: 2.2017 q80: 2.3455 q90: 2.6130 q100: 4.2616\n",
  1018. "[one_class_0 CSI 0.5641] [one_class_0 best 0.5641] \n",
  1019. "[one_class_mean CSI 0.5641] [one_class_mean best 0.5641] \n",
  1020. "0.5641\t0.5641\n"
  1021. ]
  1022. }
  1023. ],
  1024. "source": [
  1025. "# EVALUATION\n",
  1026. "# dataset : CNMC\n",
  1027. "# res : 450px\n",
  1028. "# id_class : hem\n",
  1029. "# epoch : 100\n",
  1030. "# shift_tr : randpers\n",
  1031. "# crop : 0.08\n",
  1032. "# randpers : 0.9\n",
  1033. "# color_dist : 0.5\n",
  1034. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --distortion_scale 0.9 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.9_one_class_1/last.model\""
  1035. ]
  1036. },
  1037. {
  1038. "cell_type": "code",
  1039. "execution_count": 39,
  1040. "id": "1205e882",
  1041. "metadata": {},
  1042. "outputs": [
  1043. {
  1044. "name": "stdout",
  1045. "output_type": "stream",
  1046. "text": [
  1047. "Pre-compute global statistics...\n",
  1048. "axis size: 3581 3581 3581 3581\n",
  1049. "weight_sim:\t0.0034\t0.0047\t0.0029\t0.0041\n",
  1050. "weight_shi:\t0.1303\t-0.3875\t-0.1777\t-0.3820\n",
  1051. "Pre-compute features...\n",
  1052. "Compute OOD scores... (score: CSI)\n",
  1053. "One_class_real_mean: 0.34714879632161555\n",
  1054. "CNMC 1.9226 +- 0.3041 q0: 0.5031 q10: 1.4958 q20: 1.7161 q30: 1.8443 q40: 1.9245 q50: 1.9879 q60: 2.0373 q70: 2.0889 q80: 2.1566 q90: 2.2287 q100: 2.5521\n",
  1055. "one_class_0 2.0685 +- 0.2043 q0: 1.1682 q10: 1.8203 q20: 1.9473 q30: 2.0089 q40: 2.0513 q50: 2.0872 q60: 2.1284 q70: 2.1701 q80: 2.2162 q90: 2.2834 q100: 2.6224\n",
  1056. "[one_class_0 CSI 0.3471] [one_class_0 best 0.3471] \n",
  1057. "[one_class_mean CSI 0.3471] [one_class_mean best 0.3471] \n",
  1058. "0.3471\t0.3471\n"
  1059. ]
  1060. }
  1061. ],
  1062. "source": [
  1063. "# EVALUATION\n",
  1064. "# dataset : CNMC\n",
  1065. "# res : 450px\n",
  1066. "# id_class : hem\n",
  1067. "# epoch : 100\n",
  1068. "# shift_tr : randpers\n",
  1069. "# crop : 0.08\n",
  1070. "# randpers : 0.85\n",
  1071. "# color_dist : 0.5\n",
  1072. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --distortion_scale 0.85 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.85_one_class_1/last.model\""
  1073. ]
  1074. },
  1075. {
  1076. "cell_type": "code",
  1077. "execution_count": 40,
  1078. "id": "8887546c",
  1079. "metadata": {},
  1080. "outputs": [
  1081. {
  1082. "name": "stdout",
  1083. "output_type": "stream",
  1084. "text": [
  1085. "Pre-compute global statistics...\n",
  1086. "axis size: 3581 3581 3581 3581\n",
  1087. "weight_sim:\t0.0020\t0.0037\t0.0026\t0.0039\n",
  1088. "weight_shi:\t0.1393\t2.5299\t-1.4218\t1.2437\n",
  1089. "Pre-compute features...\n",
  1090. "Compute OOD scores... (score: CSI)\n",
  1091. "One_class_real_mean: 0.6913884078226435\n",
  1092. "CNMC 2.0752 +- 0.3123 q0: 0.7320 q10: 1.7083 q20: 1.8297 q30: 1.9220 q40: 2.0047 q50: 2.0690 q60: 2.1391 q70: 2.2162 q80: 2.3010 q90: 2.4549 q100: 3.2842\n",
  1093. "one_class_0 1.8917 +- 0.2289 q0: 0.7422 q10: 1.6150 q20: 1.7197 q30: 1.7818 q40: 1.8380 q50: 1.8923 q60: 1.9400 q70: 1.9926 q80: 2.0616 q90: 2.1731 q100: 2.9070\n",
  1094. "[one_class_0 CSI 0.6914] [one_class_0 best 0.6914] \n",
  1095. "[one_class_mean CSI 0.6914] [one_class_mean best 0.6914] \n",
  1096. "0.6914\t0.6914\n"
  1097. ]
  1098. }
  1099. ],
  1100. "source": [
  1101. "# EVALUATION\n",
  1102. "# dataset : CNMC\n",
  1103. "# res : 450px\n",
  1104. "# id_class : hem\n",
  1105. "# epoch : 100\n",
  1106. "# shift_tr : randpers\n",
  1107. "# crop : 0.08\n",
  1108. "# randpers : 0.8\n",
  1109. "# color_dist : 0.5\n",
  1110. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --distortion_scale 0.8 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.8_one_class_1/last.model\""
  1111. ]
  1112. },
  1113. {
  1114. "cell_type": "code",
  1115. "execution_count": 41,
  1116. "id": "b65d2295",
  1117. "metadata": {},
  1118. "outputs": [
  1119. {
  1120. "name": "stdout",
  1121. "output_type": "stream",
  1122. "text": [
  1123. "Pre-compute global statistics...\n",
  1124. "axis size: 3581 3581 3581 3581\n",
  1125. "weight_sim:\t0.0080\t0.0036\t0.0038\t0.0054\n",
  1126. "weight_shi:\t-0.0669\t-0.5647\t-0.7888\t0.5885\n",
  1127. "Pre-compute features...\n",
  1128. "Compute OOD scores... (score: CSI)\n",
  1129. "One_class_real_mean: 0.35124039133473095\n",
  1130. "CNMC 1.9592 +- 0.1769 q0: 1.1540 q10: 1.7506 q20: 1.8287 q30: 1.8809 q40: 1.9227 q50: 1.9622 q60: 1.9930 q70: 2.0505 q80: 2.0959 q90: 2.1715 q100: 2.7656\n",
  1131. "one_class_0 2.0376 +- 0.1423 q0: 1.0198 q10: 1.8838 q20: 1.9409 q30: 1.9806 q40: 2.0144 q50: 2.0407 q60: 2.0708 q70: 2.0989 q80: 2.1348 q90: 2.1967 q100: 2.6480\n",
  1132. "[one_class_0 CSI 0.3512] [one_class_0 best 0.3512] \n",
  1133. "[one_class_mean CSI 0.3512] [one_class_mean best 0.3512] \n",
  1134. "0.3512\t0.3512\n"
  1135. ]
  1136. }
  1137. ],
  1138. "source": [
  1139. "# EVALUATION\n",
  1140. "# dataset : CNMC\n",
  1141. "# res : 450px\n",
  1142. "# id_class : hem\n",
  1143. "# epoch : 100\n",
  1144. "# shift_tr : randpers\n",
  1145. "# crop : 0.08\n",
  1146. "# randpers : 0.75\n",
  1147. "# color_dist : 0.5\n",
  1148. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --distortion_scale 0.75 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.75_one_class_1/last.model\""
  1149. ]
  1150. },
  1151. {
  1152. "cell_type": "code",
  1153. "execution_count": 42,
  1154. "id": "2a818378",
  1155. "metadata": {},
  1156. "outputs": [
  1157. {
  1158. "name": "stdout",
  1159. "output_type": "stream",
  1160. "text": [
  1161. "Pre-compute global statistics...\n",
  1162. "axis size: 3581 3581 3581 3581\n",
  1163. "weight_sim:\t0.0034\t0.0037\t0.0024\t0.0028\n",
  1164. "weight_shi:\t0.5181\t-2.5612\t-0.2828\t-0.4473\n",
  1165. "Pre-compute features...\n",
  1166. "Compute OOD scores... (score: CSI)\n",
  1167. "One_class_real_mean: 0.3594818156959256\n",
  1168. "CNMC 1.5977 +- 1.3081 q0: -2.9399 q10: -0.2131 q20: 0.5889 q30: 1.0500 q40: 1.4828 q50: 1.7567 q60: 2.0337 q70: 2.3642 q80: 2.6741 q90: 3.1919 q100: 4.5132\n",
  1169. "one_class_0 2.2261 +- 1.0824 q0: -1.7685 q10: 0.7646 q20: 1.4013 q30: 1.7643 q40: 2.0621 q50: 2.3193 q60: 2.5929 q70: 2.8838 q80: 3.1160 q90: 3.5429 q100: 5.0474\n",
  1170. "[one_class_0 CSI 0.3595] [one_class_0 best 0.3595] \n",
  1171. "[one_class_mean CSI 0.3595] [one_class_mean best 0.3595] \n",
  1172. "0.3595\t0.3595\n"
  1173. ]
  1174. }
  1175. ],
  1176. "source": [
  1177. "# EVALUATION\n",
  1178. "# dataset : CNMC\n",
  1179. "# res : 450px\n",
  1180. "# id_class : hem\n",
  1181. "# epoch : 100\n",
  1182. "# shift_tr : randpers\n",
  1183. "# crop : 0.08\n",
  1184. "# randpers : 0.6\n",
  1185. "# color_dist : 0.5\n",
  1186. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --distortion_scale 0.6 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.6_one_class_1/last.model\""
  1187. ]
  1188. },
  1189. {
  1190. "cell_type": "code",
  1191. "execution_count": 43,
  1192. "id": "09a15dda",
  1193. "metadata": {},
  1194. "outputs": [
  1195. {
  1196. "name": "stdout",
  1197. "output_type": "stream",
  1198. "text": [
  1199. "Pre-compute global statistics...\n",
  1200. "axis size: 3581 3581 3581 3581\n",
  1201. "weight_sim:\t0.0043\t0.0115\t0.0075\t0.0087\n",
  1202. "weight_shi:\t12.1609\t0.3968\t2.0101\t0.4812\n",
  1203. "Pre-compute features...\n",
  1204. "Compute OOD scores... (score: CSI)\n",
  1205. "One_class_real_mean: 0.36039204367068733\n",
  1206. "CNMC 1.4531 +- 7.5510 q0: -28.7074 q10: -8.1564 q20: -3.7419 q30: -1.3692 q40: 0.5754 q50: 2.0721 q60: 3.7007 q70: 5.1638 q80: 7.3805 q90: 10.4019 q100: 20.0002\n",
  1207. "one_class_0 4.8084 +- 5.1144 q0: -14.2655 q10: -1.2285 q20: 1.1701 q30: 2.5734 q40: 3.7262 q50: 4.8972 q60: 5.9430 q70: 7.0902 q80: 8.7584 q90: 11.3302 q100: 19.8412\n",
  1208. "[one_class_0 CSI 0.3604] [one_class_0 best 0.3604] \n",
  1209. "[one_class_mean CSI 0.3604] [one_class_mean best 0.3604] \n",
  1210. "0.3604\t0.3604\n"
  1211. ]
  1212. }
  1213. ],
  1214. "source": [
  1215. "# EVALUATION\n",
  1216. "# dataset : CNMC\n",
  1217. "# res : 450px\n",
  1218. "# id_class : hem\n",
  1219. "# epoch : 100\n",
  1220. "# shift_tr : randpers\n",
  1221. "# crop : 0.08\n",
  1222. "# randpers : 0.3\n",
  1223. "# color_dist : 0.5\n",
  1224. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --distortion_scale 0.3 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.3_one_class_1/last.model\""
  1225. ]
  1226. },
  1227. {
  1228. "cell_type": "markdown",
  1229. "id": "47013663",
  1230. "metadata": {},
  1231. "source": [
  1232. "## blur"
  1233. ]
  1234. },
  1235. {
  1236. "cell_type": "code",
  1237. "execution_count": 134,
  1238. "id": "958ecba3",
  1239. "metadata": {
  1240. "scrolled": true
  1241. },
  1242. "outputs": [
  1243. {
  1244. "name": "stdout",
  1245. "output_type": "stream",
  1246. "text": [
  1247. "Pre-compute global statistics...\n",
  1248. "axis size: 3581 3581 3581 3581\n",
  1249. "weight_sim:\t0.0038\t0.0072\t0.0039\t0.0044\n",
  1250. "weight_shi:\t-0.1658\t0.1714\t0.2799\t0.2838\n",
  1251. "Pre-compute features...\n",
  1252. "Compute OOD scores... (score: CSI)\n",
  1253. "One_class_real_mean: 0.4812004375170905\n",
  1254. "CNMC 1.9384 +- 0.2410 q0: 1.5304 q10: 1.7013 q20: 1.7508 q30: 1.7894 q40: 1.8299 q50: 1.8827 q60: 1.9243 q70: 1.9999 q80: 2.0841 q90: 2.2637 q100: 2.8485\n",
  1255. "one_class_0 1.9219 +- 0.1651 q0: 1.5451 q10: 1.7386 q20: 1.7816 q30: 1.8174 q40: 1.8548 q50: 1.8945 q60: 1.9407 q70: 1.9846 q80: 2.0549 q90: 2.1446 q100: 2.6371\n",
  1256. "[one_class_0 CSI 0.4812] [one_class_0 best 0.4812] \n",
  1257. "[one_class_mean CSI 0.4812] [one_class_mean best 0.4812] \n",
  1258. "0.4812\t0.4812\n"
  1259. ]
  1260. }
  1261. ],
  1262. "source": [
  1263. "# EVALUATION\n",
  1264. "# dataset : CNMC\n",
  1265. "# res : 450px\n",
  1266. "# id_class : hem\n",
  1267. "# epoch : 100\n",
  1268. "# shift_tr : blur\n",
  1269. "# crop : 0.08\n",
  1270. "# blur_sigma : 180\n",
  1271. "# color_dist : 0.5\n",
  1272. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 180 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma180.0_one_class_1/last.model\""
  1273. ]
  1274. },
  1275. {
  1276. "cell_type": "code",
  1277. "execution_count": 135,
  1278. "id": "a3f7ef72",
  1279. "metadata": {},
  1280. "outputs": [
  1281. {
  1282. "name": "stdout",
  1283. "output_type": "stream",
  1284. "text": [
  1285. "Pre-compute global statistics...\n",
  1286. "axis size: 3581 3581 3581 3581\n",
  1287. "weight_sim:\t0.0025\t0.0058\t0.0024\t0.0029\n",
  1288. "weight_shi:\t-0.0568\t0.0831\t0.1701\t0.1303\n",
  1289. "Pre-compute features...\n",
  1290. "Compute OOD scores... (score: CSI)\n",
  1291. "One_class_real_mean: 0.6192537902956279\n",
  1292. "CNMC 2.0247 +- 0.0991 q0: 1.8346 q10: 1.9155 q20: 1.9404 q30: 1.9652 q40: 1.9879 q50: 2.0067 q60: 2.0291 q70: 2.0625 q80: 2.1019 q90: 2.1563 q100: 2.4786\n",
  1293. "one_class_0 1.9853 +- 0.0765 q0: 1.7917 q10: 1.9064 q20: 1.9276 q30: 1.9429 q40: 1.9598 q50: 1.9743 q60: 1.9887 q70: 2.0055 q80: 2.0343 q90: 2.0845 q100: 2.3701\n",
  1294. "[one_class_0 CSI 0.6193] [one_class_0 best 0.6193] \n",
  1295. "[one_class_mean CSI 0.6193] [one_class_mean best 0.6193] \n",
  1296. "0.6193\t0.6193\n"
  1297. ]
  1298. }
  1299. ],
  1300. "source": [
  1301. "# EVALUATION\n",
  1302. "# dataset : CNMC\n",
  1303. "# res : 450px\n",
  1304. "# id_class : hem\n",
  1305. "# epoch : 100\n",
  1306. "# shift_tr : blur\n",
  1307. "# crop : 0.08\n",
  1308. "# blur_sigma : 120\n",
  1309. "# color_dist : 0.5\n",
  1310. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 120 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma120.0_one_class_1/last.model\""
  1311. ]
  1312. },
  1313. {
  1314. "cell_type": "code",
  1315. "execution_count": 147,
  1316. "id": "2f2a8808",
  1317. "metadata": {},
  1318. "outputs": [
  1319. {
  1320. "name": "stdout",
  1321. "output_type": "stream",
  1322. "text": [
  1323. "Pre-compute global statistics...\n",
  1324. "axis size: 3581 3581 3581 3581\n",
  1325. "weight_sim:\t0.0030\t0.0043\t0.0026\t0.0028\n",
  1326. "weight_shi:\t-0.0889\t0.1756\t0.3138\t0.2610\n",
  1327. "Pre-compute features...\n",
  1328. "Compute OOD scores... (score: CSI)\n",
  1329. "One_class_real_mean: 0.5952460527248604\n",
  1330. "CNMC 2.0008 +- 0.0139 q0: 1.9745 q10: 1.9866 q20: 1.9900 q30: 1.9930 q40: 1.9955 q50: 1.9986 q60: 2.0019 q70: 2.0048 q80: 2.0099 q90: 2.0166 q100: 2.0896\n",
  1331. "one_class_0 1.9964 +- 0.0119 q0: 1.9575 q10: 1.9833 q20: 1.9872 q30: 1.9899 q40: 1.9925 q50: 1.9948 q60: 1.9973 q70: 2.0007 q80: 2.0051 q90: 2.0119 q100: 2.0732\n",
  1332. "[one_class_0 CSI 0.5952] [one_class_0 best 0.5952] \n",
  1333. "[one_class_mean CSI 0.5952] [one_class_mean best 0.5952] \n",
  1334. "0.5952\t0.5952\n"
  1335. ]
  1336. }
  1337. ],
  1338. "source": [
  1339. "# EVALUATION\n",
  1340. "# dataset : CNMC\n",
  1341. "# res : 450px\n",
  1342. "# id_class : hem\n",
  1343. "# epoch : 100\n",
  1344. "# shift_tr : blur\n",
  1345. "# crop : 0.08\n",
  1346. "# blur_sigma : 110\n",
  1347. "# color_dist : 0.5\n",
  1348. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 110 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma110.0_one_class_1/last.model\""
  1349. ]
  1350. },
  1351. {
  1352. "cell_type": "code",
  1353. "execution_count": 136,
  1354. "id": "08a6959c",
  1355. "metadata": {},
  1356. "outputs": [
  1357. {
  1358. "name": "stdout",
  1359. "output_type": "stream",
  1360. "text": [
  1361. "Pre-compute global statistics...\n",
  1362. "axis size: 3581 3581 3581 3581\n",
  1363. "weight_sim:\t0.0067\t0.0043\t0.0047\t0.0049\n",
  1364. "weight_shi:\t-0.0583\t0.0915\t0.2051\t0.1700\n",
  1365. "Pre-compute features...\n",
  1366. "Compute OOD scores... (score: CSI)\n",
  1367. "One_class_real_mean: 0.633952896018797\n",
  1368. "CNMC 1.9904 +- 0.1692 q0: 1.5494 q10: 1.7849 q20: 1.8442 q30: 1.9005 q40: 1.9430 q50: 1.9786 q60: 2.0203 q70: 2.0602 q80: 2.1209 q90: 2.2064 q100: 2.5511\n",
  1369. "one_class_0 1.9167 +- 0.1098 q0: 1.5787 q10: 1.7796 q20: 1.8329 q30: 1.8646 q40: 1.8922 q50: 1.9150 q60: 1.9402 q70: 1.9662 q80: 1.9991 q90: 2.0546 q100: 2.2965\n",
  1370. "[one_class_0 CSI 0.6340] [one_class_0 best 0.6340] \n",
  1371. "[one_class_mean CSI 0.6340] [one_class_mean best 0.6340] \n",
  1372. "0.6340\t0.6340\n"
  1373. ]
  1374. }
  1375. ],
  1376. "source": [
  1377. "# EVALUATION\n",
  1378. "# dataset : CNMC\n",
  1379. "# res : 450px\n",
  1380. "# id_class : hem\n",
  1381. "# epoch : 100\n",
  1382. "# shift_tr : blur\n",
  1383. "# crop : 0.08\n",
  1384. "# blur_sigma : 105\n",
  1385. "# color_dist : 0.5\n",
  1386. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 105 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma105.0_one_class_1/last.model\""
  1387. ]
  1388. },
  1389. {
  1390. "cell_type": "code",
  1391. "execution_count": 137,
  1392. "id": "a4a4eee1",
  1393. "metadata": {},
  1394. "outputs": [
  1395. {
  1396. "name": "stdout",
  1397. "output_type": "stream",
  1398. "text": [
  1399. "Pre-compute global statistics...\n",
  1400. "axis size: 3581 3581 3581 3581\n",
  1401. "weight_sim:\t0.0091\t0.0045\t0.0091\t0.0070\n",
  1402. "weight_shi:\t-0.0676\t0.0975\t0.1849\t0.1972\n",
  1403. "Pre-compute features...\n",
  1404. "Compute OOD scores... (score: CSI)\n",
  1405. "One_class_real_mean: 0.5476483456385015\n",
  1406. "CNMC 1.9769 +- 0.2661 q0: 1.3171 q10: 1.6312 q20: 1.7272 q30: 1.8319 q40: 1.9118 q50: 1.9776 q60: 2.0460 q70: 2.1125 q80: 2.1924 q90: 2.3278 q100: 2.8173\n",
  1407. "one_class_0 1.9268 +- 0.2241 q0: 1.2928 q10: 1.6165 q20: 1.7138 q30: 1.8152 q40: 1.8859 q50: 1.9465 q60: 2.0048 q70: 2.0580 q80: 2.1179 q90: 2.1973 q100: 2.5704\n",
  1408. "[one_class_0 CSI 0.5476] [one_class_0 best 0.5476] \n",
  1409. "[one_class_mean CSI 0.5476] [one_class_mean best 0.5476] \n",
  1410. "0.5476\t0.5476\n"
  1411. ]
  1412. }
  1413. ],
  1414. "source": [
  1415. "# EVALUATION\n",
  1416. "# dataset : CNMC\n",
  1417. "# res : 450px\n",
  1418. "# id_class : hem\n",
  1419. "# epoch : 100\n",
  1420. "# shift_tr : blur\n",
  1421. "# crop : 0.08\n",
  1422. "# blur_sigma : 100\n",
  1423. "# color_dist : 0.5\n",
  1424. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 100 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma100.0_one_class_1/last.model\""
  1425. ]
  1426. },
  1427. {
  1428. "cell_type": "code",
  1429. "execution_count": 138,
  1430. "id": "8f0ceb15",
  1431. "metadata": {
  1432. "scrolled": true
  1433. },
  1434. "outputs": [
  1435. {
  1436. "name": "stdout",
  1437. "output_type": "stream",
  1438. "text": [
  1439. "Pre-compute global statistics...\n",
  1440. "axis size: 3581 3581 3581 3581\n",
  1441. "weight_sim:\t0.0018\t0.0028\t0.0016\t0.0018\n",
  1442. "weight_shi:\t-0.2029\t0.1970\t1.0597\t0.4185\n",
  1443. "Pre-compute features...\n",
  1444. "Compute OOD scores... (score: CSI)\n",
  1445. "One_class_real_mean: 0.5393334953767002\n",
  1446. "CNMC 1.9749 +- 0.3046 q0: 1.1986 q10: 1.5960 q20: 1.7089 q30: 1.8060 q40: 1.8905 q50: 1.9696 q60: 2.0356 q70: 2.1233 q80: 2.2217 q90: 2.3752 q100: 3.1061\n",
  1447. "one_class_0 1.9275 +- 0.2387 q0: 1.1897 q10: 1.6222 q20: 1.7266 q30: 1.8084 q40: 1.8732 q50: 1.9343 q60: 1.9958 q70: 2.0464 q80: 2.1172 q90: 2.2327 q100: 2.6893\n",
  1448. "[one_class_0 CSI 0.5393] [one_class_0 best 0.5393] \n",
  1449. "[one_class_mean CSI 0.5393] [one_class_mean best 0.5393] \n",
  1450. "0.5393\t0.5393\n"
  1451. ]
  1452. }
  1453. ],
  1454. "source": [
  1455. "# EVALUATION\n",
  1456. "# dataset : CNMC\n",
  1457. "# res : 450px\n",
  1458. "# id_class : hem\n",
  1459. "# epoch : 100\n",
  1460. "# shift_tr : blur\n",
  1461. "# crop : 0.08\n",
  1462. "# blur_sigma : 95\n",
  1463. "# color_dist : 0.5\n",
  1464. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 95 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma95.0_one_class_1/last.model\""
  1465. ]
  1466. },
  1467. {
  1468. "cell_type": "code",
  1469. "execution_count": 139,
  1470. "id": "7d89e279",
  1471. "metadata": {
  1472. "scrolled": true
  1473. },
  1474. "outputs": [
  1475. {
  1476. "name": "stdout",
  1477. "output_type": "stream",
  1478. "text": [
  1479. "Pre-compute global statistics...\n",
  1480. "axis size: 3581 3581 3581 3581\n",
  1481. "weight_sim:\t0.0075\t0.0041\t0.0071\t0.0059\n",
  1482. "weight_shi:\t-0.0360\t0.0714\t0.1079\t0.0991\n",
  1483. "Pre-compute features...\n",
  1484. "Compute OOD scores... (score: CSI)\n",
  1485. "One_class_real_mean: 0.5292488277175179\n",
  1486. "CNMC 1.9774 +- 0.1920 q0: 1.5068 q10: 1.7357 q20: 1.8106 q30: 1.8745 q40: 1.9206 q50: 1.9753 q60: 2.0197 q70: 2.0694 q80: 2.1299 q90: 2.2192 q100: 2.6272\n",
  1487. "one_class_0 1.9507 +- 0.1545 q0: 1.4789 q10: 1.7436 q20: 1.8103 q30: 1.8750 q40: 1.9239 q50: 1.9683 q60: 2.0050 q70: 2.0411 q80: 2.0780 q90: 2.1379 q100: 2.3968\n",
  1488. "[one_class_0 CSI 0.5292] [one_class_0 best 0.5292] \n",
  1489. "[one_class_mean CSI 0.5292] [one_class_mean best 0.5292] \n",
  1490. "0.5292\t0.5292\n"
  1491. ]
  1492. }
  1493. ],
  1494. "source": [
  1495. "# EVALUATION\n",
  1496. "# dataset : CNMC\n",
  1497. "# res : 450px\n",
  1498. "# id_class : hem\n",
  1499. "# epoch : 100\n",
  1500. "# shift_tr : blur\n",
  1501. "# crop : 0.08\n",
  1502. "# blur_sigma : 90\n",
  1503. "# color_dist : 0.5\n",
  1504. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 90 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma90.0_one_class_1/last.model\""
  1505. ]
  1506. },
  1507. {
  1508. "cell_type": "code",
  1509. "execution_count": 140,
  1510. "id": "ebb47e6b",
  1511. "metadata": {},
  1512. "outputs": [
  1513. {
  1514. "name": "stdout",
  1515. "output_type": "stream",
  1516. "text": [
  1517. "Pre-compute global statistics...\n",
  1518. "axis size: 3581 3581 3581 3581\n",
  1519. "weight_sim:\t0.0050\t0.0117\t0.0038\t0.0049\n",
  1520. "weight_shi:\t-0.2427\t0.2328\t1.3692\t0.7248\n",
  1521. "Pre-compute features...\n",
  1522. "Compute OOD scores... (score: CSI)\n",
  1523. "One_class_real_mean: 0.557083573866456\n",
  1524. "CNMC 2.0005 +- 0.1926 q0: 1.3343 q10: 1.8135 q20: 1.8631 q30: 1.9060 q40: 1.9410 q50: 1.9699 q60: 2.0110 q70: 2.0589 q80: 2.1204 q90: 2.2186 q100: 3.1284\n",
  1525. "one_class_0 1.9634 +- 0.1487 q0: 1.4064 q10: 1.8025 q20: 1.8544 q30: 1.8886 q40: 1.9192 q50: 1.9463 q60: 1.9749 q70: 2.0096 q80: 2.0594 q90: 2.1522 q100: 2.5877\n",
  1526. "[one_class_0 CSI 0.5571] [one_class_0 best 0.5571] \n",
  1527. "[one_class_mean CSI 0.5571] [one_class_mean best 0.5571] \n",
  1528. "0.5571\t0.5571\n"
  1529. ]
  1530. }
  1531. ],
  1532. "source": [
  1533. "# EVALUATION\n",
  1534. "# dataset : CNMC\n",
  1535. "# res : 450px\n",
  1536. "# id_class : hem\n",
  1537. "# epoch : 100\n",
  1538. "# shift_tr : blur\n",
  1539. "# crop : 0.08\n",
  1540. "# blur_sigma : 80\n",
  1541. "# color_dist : 0.5\n",
  1542. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 80 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma80.0_one_class_1/last.model\""
  1543. ]
  1544. },
  1545. {
  1546. "cell_type": "code",
  1547. "execution_count": 141,
  1548. "id": "7d6e0050",
  1549. "metadata": {
  1550. "scrolled": true
  1551. },
  1552. "outputs": [
  1553. {
  1554. "name": "stdout",
  1555. "output_type": "stream",
  1556. "text": [
  1557. "Pre-compute global statistics...\n",
  1558. "axis size: 3581 3581 3581 3581\n",
  1559. "weight_sim:\t0.0062\t0.0053\t0.0066\t0.0062\n",
  1560. "weight_shi:\t-0.0434\t0.0771\t0.1221\t0.1065\n",
  1561. "Pre-compute features...\n",
  1562. "Compute OOD scores... (score: CSI)\n",
  1563. "One_class_real_mean: 0.5821104122990916\n",
  1564. "CNMC 1.9984 +- 0.0869 q0: 1.7841 q10: 1.8832 q20: 1.9216 q30: 1.9505 q40: 1.9768 q50: 1.9991 q60: 2.0230 q70: 2.0443 q80: 2.0710 q90: 2.1126 q100: 2.2334\n",
  1565. "one_class_0 1.9740 +- 0.0685 q0: 1.7594 q10: 1.8780 q20: 1.9143 q30: 1.9428 q40: 1.9641 q50: 1.9808 q60: 1.9973 q70: 2.0131 q80: 2.0305 q90: 2.0551 q100: 2.1770\n",
  1566. "[one_class_0 CSI 0.5821] [one_class_0 best 0.5821] \n",
  1567. "[one_class_mean CSI 0.5821] [one_class_mean best 0.5821] \n",
  1568. "0.5821\t0.5821\n"
  1569. ]
  1570. }
  1571. ],
  1572. "source": [
  1573. "# EVALUATION\n",
  1574. "# dataset : CNMC\n",
  1575. "# res : 450px\n",
  1576. "# id_class : hem\n",
  1577. "# epoch : 100\n",
  1578. "# shift_tr : blur\n",
  1579. "# crop : 0.08\n",
  1580. "# blur_sigma : 60\n",
  1581. "# color_dist : 0.5\n",
  1582. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 60 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma60.0_one_class_1/last.model\""
  1583. ]
  1584. },
  1585. {
  1586. "cell_type": "code",
  1587. "execution_count": 142,
  1588. "id": "df7becce",
  1589. "metadata": {},
  1590. "outputs": [
  1591. {
  1592. "name": "stdout",
  1593. "output_type": "stream",
  1594. "text": [
  1595. "Pre-compute global statistics...\n",
  1596. "axis size: 3581 3581 3581 3581\n",
  1597. "weight_sim:\t0.0045\t0.0051\t0.0033\t0.0041\n",
  1598. "weight_shi:\t-0.1512\t0.2745\t0.6510\t0.4026\n",
  1599. "Pre-compute features...\n",
  1600. "Compute OOD scores... (score: CSI)\n",
  1601. "One_class_real_mean: 0.6940311072625811\n",
  1602. "CNMC 2.0096 +- 0.0626 q0: 1.8488 q10: 1.9389 q20: 1.9597 q30: 1.9741 q40: 1.9884 q50: 2.0031 q60: 2.0185 q70: 2.0342 q80: 2.0520 q90: 2.0853 q100: 2.2770\n",
  1603. "one_class_0 1.9718 +- 0.0446 q0: 1.8450 q10: 1.9219 q20: 1.9383 q30: 1.9495 q40: 1.9584 q50: 1.9680 q60: 1.9769 q70: 1.9880 q80: 2.0015 q90: 2.0244 q100: 2.2023\n",
  1604. "[one_class_0 CSI 0.6940] [one_class_0 best 0.6940] \n",
  1605. "[one_class_mean CSI 0.6940] [one_class_mean best 0.6940] \n",
  1606. "0.6940\t0.6940\n"
  1607. ]
  1608. }
  1609. ],
  1610. "source": [
  1611. "# EVALUATION\n",
  1612. "# dataset : CNMC\n",
  1613. "# res : 450px\n",
  1614. "# id_class : hem\n",
  1615. "# epoch : 100\n",
  1616. "# shift_tr : blur\n",
  1617. "# crop : 0.08\n",
  1618. "# blur_sigma : 40\n",
  1619. "# color_dist : 0.5\n",
  1620. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 40 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma40.0_one_class_1/last.model\""
  1621. ]
  1622. },
  1623. {
  1624. "cell_type": "code",
  1625. "execution_count": 143,
  1626. "id": "b7036b42",
  1627. "metadata": {},
  1628. "outputs": [
  1629. {
  1630. "name": "stdout",
  1631. "output_type": "stream",
  1632. "text": [
  1633. "Pre-compute global statistics...\n",
  1634. "axis size: 3581 3581 3581 3581\n",
  1635. "weight_sim:\t0.0017\t0.0020\t0.0015\t0.0016\n",
  1636. "weight_shi:\t0.0317\t-0.1164\t-0.0840\t-0.0812\n",
  1637. "Pre-compute features...\n",
  1638. "Compute OOD scores... (score: CSI)\n",
  1639. "One_class_real_mean: 0.39877986408612603\n",
  1640. "CNMC 1.9759 +- 0.1316 q0: 1.4471 q10: 1.8010 q20: 1.8983 q30: 1.9382 q40: 1.9698 q50: 1.9932 q60: 2.0241 q70: 2.0528 q80: 2.0799 q90: 2.1197 q100: 2.2278\n",
  1641. "one_class_0 2.0210 +- 0.0942 q0: 1.5614 q10: 1.8968 q20: 1.9555 q30: 1.9874 q40: 2.0148 q50: 2.0364 q60: 2.0551 q70: 2.0753 q80: 2.0963 q90: 2.1246 q100: 2.2320\n",
  1642. "[one_class_0 CSI 0.3988] [one_class_0 best 0.3988] \n",
  1643. "[one_class_mean CSI 0.3988] [one_class_mean best 0.3988] \n",
  1644. "0.3988\t0.3988\n"
  1645. ]
  1646. }
  1647. ],
  1648. "source": [
  1649. "# EVALUATION\n",
  1650. "# dataset : CNMC\n",
  1651. "# res : 450px\n",
  1652. "# id_class : hem\n",
  1653. "# epoch : 100\n",
  1654. "# shift_tr : blur\n",
  1655. "# crop : 0.08\n",
  1656. "# blur_sigma : 20\n",
  1657. "# color_dist : 0.5\n",
  1658. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 20 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma20.0_one_class_1/last.model\""
  1659. ]
  1660. },
  1661. {
  1662. "cell_type": "code",
  1663. "execution_count": 144,
  1664. "id": "e7b68654",
  1665. "metadata": {},
  1666. "outputs": [
  1667. {
  1668. "name": "stdout",
  1669. "output_type": "stream",
  1670. "text": [
  1671. "Pre-compute global statistics...\n",
  1672. "axis size: 3581 3581 3581 3581\n",
  1673. "weight_sim:\t0.0020\t0.0035\t0.0027\t0.0027\n",
  1674. "weight_shi:\t0.1013\t-0.5641\t-0.5419\t-0.3880\n",
  1675. "Pre-compute features...\n",
  1676. "Compute OOD scores... (score: CSI)\n",
  1677. "One_class_real_mean: 0.33394542683235595\n",
  1678. "CNMC 1.9739 +- 0.2803 q0: 1.2318 q10: 1.5933 q20: 1.7359 q30: 1.8196 q40: 1.9141 q50: 1.9863 q60: 2.0640 q70: 2.1425 q80: 2.2339 q90: 2.3203 q100: 2.6037\n",
  1679. "one_class_0 2.1309 +- 0.1875 q0: 1.4830 q10: 1.8910 q20: 1.9714 q30: 2.0347 q40: 2.0807 q50: 2.1311 q60: 2.1754 q70: 2.2372 q80: 2.3011 q90: 2.3743 q100: 2.6831\n",
  1680. "[one_class_0 CSI 0.3339] [one_class_0 best 0.3339] \n",
  1681. "[one_class_mean CSI 0.3339] [one_class_mean best 0.3339] \n",
  1682. "0.3339\t0.3339\n"
  1683. ]
  1684. }
  1685. ],
  1686. "source": [
  1687. "# EVALUATION\n",
  1688. "# dataset : CNMC\n",
  1689. "# shift_tr : blur\n",
  1690. "# id_class : hem\n",
  1691. "# epoch : 100\n",
  1692. "# res : 450px\n",
  1693. "# crop : 0.08\n",
  1694. "# blur_sigma : 6\n",
  1695. "# color_dist : 0.5\n",
  1696. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 6 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma6.0_one_class_1/last.model\""
  1697. ]
  1698. },
  1699. {
  1700. "cell_type": "code",
  1701. "execution_count": 145,
  1702. "id": "5a20ddb8",
  1703. "metadata": {},
  1704. "outputs": [
  1705. {
  1706. "name": "stdout",
  1707. "output_type": "stream",
  1708. "text": [
  1709. "Pre-compute global statistics...\n",
  1710. "axis size: 3581 3581 3581 3581\n",
  1711. "weight_sim:\t0.0029\t0.0062\t0.0034\t0.0030\n",
  1712. "weight_shi:\t0.2169\t2.1291\t-0.6997\t-0.6317\n",
  1713. "Pre-compute features...\n",
  1714. "Compute OOD scores... (score: CSI)\n",
  1715. "One_class_real_mean: 0.5895074388033098\n",
  1716. "CNMC 2.0596 +- 0.2877 q0: 1.0222 q10: 1.6929 q20: 1.8375 q30: 1.9167 q40: 2.0096 q50: 2.0851 q60: 2.1435 q70: 2.2065 q80: 2.2951 q90: 2.4044 q100: 3.0604\n",
  1717. "one_class_0 1.9839 +- 0.2326 q0: 1.1067 q10: 1.6760 q20: 1.7908 q30: 1.8756 q40: 1.9429 q50: 2.0082 q60: 2.0584 q70: 2.1101 q80: 2.1732 q90: 2.2584 q100: 3.0116\n",
  1718. "[one_class_0 CSI 0.5895] [one_class_0 best 0.5895] \n",
  1719. "[one_class_mean CSI 0.5895] [one_class_mean best 0.5895] \n",
  1720. "0.5895\t0.5895\n"
  1721. ]
  1722. }
  1723. ],
  1724. "source": [
  1725. "# EVALUATION\n",
  1726. "# dataset : CNMC\n",
  1727. "# res : 450px\n",
  1728. "# id_class : hem\n",
  1729. "# epoch : 100\n",
  1730. "# shift_tr : blur\n",
  1731. "# crop : 0.08\n",
  1732. "# blur_sigma : 4\n",
  1733. "# color_dist : 0.5\n",
  1734. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 4 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma4.0_one_class_1/last.model\""
  1735. ]
  1736. },
  1737. {
  1738. "cell_type": "code",
  1739. "execution_count": 146,
  1740. "id": "f014e06d",
  1741. "metadata": {},
  1742. "outputs": [
  1743. {
  1744. "name": "stdout",
  1745. "output_type": "stream",
  1746. "text": [
  1747. "Pre-compute global statistics...\n",
  1748. "axis size: 3581 3581 3581 3581\n",
  1749. "weight_sim:\t0.0047\t0.0065\t0.0046\t0.0045\n",
  1750. "weight_shi:\t0.2645\t-12.1918\t-1.1354\t-0.9111\n",
  1751. "Pre-compute features...\n",
  1752. "Compute OOD scores... (score: CSI)\n",
  1753. "One_class_real_mean: 0.43248488439218546\n",
  1754. "CNMC 1.5307 +- 2.2981 q0: -6.2860 q10: -1.3964 q20: -0.1077 q30: 0.5774 q40: 1.2354 q50: 1.7091 q60: 2.1039 q70: 2.7068 q80: 3.3782 q90: 4.4476 q100: 6.9377\n",
  1755. "one_class_0 2.0424 +- 1.5916 q0: -5.1678 q10: 0.0924 q20: 0.8505 q30: 1.3834 q40: 1.7445 q50: 2.1476 q60: 2.4484 q70: 2.8574 q80: 3.3216 q90: 3.9483 q100: 6.4052\n",
  1756. "[one_class_0 CSI 0.4325] [one_class_0 best 0.4325] \n",
  1757. "[one_class_mean CSI 0.4325] [one_class_mean best 0.4325] \n",
  1758. "0.4325\t0.4325\n"
  1759. ]
  1760. }
  1761. ],
  1762. "source": [
  1763. "# EVALUATION\n",
  1764. "# dataset : CNMC\n",
  1765. "# res : 450px\n",
  1766. "# id_class : hem\n",
  1767. "# epoch : 100\n",
  1768. "# shift_tr : blur\n",
  1769. "# crop : 0.08\n",
  1770. "# blur_sigma : 3\n",
  1771. "# color_dist : 0.5\n",
  1772. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 3 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma3.0_one_class_1/last.model\""
  1773. ]
  1774. },
  1775. {
  1776. "cell_type": "code",
  1777. "execution_count": 148,
  1778. "id": "469197e2",
  1779. "metadata": {},
  1780. "outputs": [
  1781. {
  1782. "name": "stdout",
  1783. "output_type": "stream",
  1784. "text": [
  1785. "Pre-compute global statistics...\n",
  1786. "axis size: 3581 3581 3581 3581\n",
  1787. "weight_sim:\t0.0083\t0.0100\t0.0115\t0.0075\n",
  1788. "weight_shi:\t2.4798\t0.7962\t-4.3631\t-2.5771\n",
  1789. "Pre-compute features...\n",
  1790. "Compute OOD scores... (score: CSI)\n",
  1791. "One_class_real_mean: 0.37173128145920054\n",
  1792. "CNMC 1.8398 +- 0.7107 q0: -1.6740 q10: 0.9741 q20: 1.3853 q30: 1.6153 q40: 1.8033 q50: 1.9486 q60: 2.1053 q70: 2.2304 q80: 2.3601 q90: 2.5741 q100: 3.5645\n",
  1793. "one_class_0 2.1409 +- 0.5323 q0: -0.7020 q10: 1.4665 q20: 1.7704 q30: 1.9274 q40: 2.0616 q50: 2.1799 q60: 2.2918 q70: 2.4188 q80: 2.5430 q90: 2.7544 q100: 3.8279\n",
  1794. "[one_class_0 CSI 0.3717] [one_class_0 best 0.3717] \n",
  1795. "[one_class_mean CSI 0.3717] [one_class_mean best 0.3717] \n",
  1796. "0.3717\t0.3717\n"
  1797. ]
  1798. }
  1799. ],
  1800. "source": [
  1801. "# EVALUATION\n",
  1802. "# dataset : CNMC\n",
  1803. "# res : 450px\n",
  1804. "# id_class : hem\n",
  1805. "# epoch : 100\n",
  1806. "# shift_tr : blur\n",
  1807. "# crop : 0.08\n",
  1808. "# blur_sigma : 2\n",
  1809. "# color_dist : 0.5\n",
  1810. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma2.0_one_class_1/last.model\""
  1811. ]
  1812. },
  1813. {
  1814. "cell_type": "code",
  1815. "execution_count": 149,
  1816. "id": "b8ccee0f",
  1817. "metadata": {},
  1818. "outputs": [
  1819. {
  1820. "name": "stdout",
  1821. "output_type": "stream",
  1822. "text": [
  1823. "Pre-compute global statistics...\n",
  1824. "axis size: 3581 3581 3581 3581\n",
  1825. "weight_sim:\t0.0078\t0.0116\t0.0095\t0.0106\n",
  1826. "weight_shi:\t0.1768\t-0.5198\t-0.4439\t-0.3696\n",
  1827. "Pre-compute features...\n",
  1828. "Compute OOD scores... (score: CSI)\n",
  1829. "One_class_real_mean: 0.3392225969475081\n",
  1830. "CNMC 1.9808 +- 0.1438 q0: 1.5735 q10: 1.8230 q20: 1.8637 q30: 1.8908 q40: 1.9230 q50: 1.9577 q60: 1.9928 q70: 2.0435 q80: 2.1120 q90: 2.1900 q100: 2.4139\n",
  1831. "one_class_0 2.0502 +- 0.1152 q0: 1.7554 q10: 1.9134 q20: 1.9501 q30: 1.9799 q40: 2.0064 q50: 2.0376 q60: 2.0668 q70: 2.1043 q80: 2.1520 q90: 2.2170 q100: 2.4357\n",
  1832. "[one_class_0 CSI 0.3392] [one_class_0 best 0.3392] \n",
  1833. "[one_class_mean CSI 0.3392] [one_class_mean best 0.3392] \n",
  1834. "0.3392\t0.3392\n"
  1835. ]
  1836. }
  1837. ],
  1838. "source": [
  1839. "# EVALUATION\n",
  1840. "# dataset : CNMC\n",
  1841. "# res : 450px\n",
  1842. "# id_class : hem\n",
  1843. "# epoch : 100\n",
  1844. "# shift_tr : blur\n",
  1845. "# crop : 0.08\n",
  1846. "# blur_sigma : 1.5\n",
  1847. "# color_dist : 0.5\n",
  1848. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 1.5 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma1.5_one_class_1/last.model\""
  1849. ]
  1850. },
  1851. {
  1852. "cell_type": "code",
  1853. "execution_count": 150,
  1854. "id": "3ba56d85",
  1855. "metadata": {
  1856. "scrolled": true
  1857. },
  1858. "outputs": [
  1859. {
  1860. "name": "stdout",
  1861. "output_type": "stream",
  1862. "text": [
  1863. "Pre-compute global statistics...\n",
  1864. "axis size: 3581 3581 3581 3581\n",
  1865. "weight_sim:\t0.0021\t0.0031\t0.0026\t0.0026\n",
  1866. "weight_shi:\t0.3756\t9.2614\t-0.9536\t-0.8326\n",
  1867. "Pre-compute features...\n",
  1868. "Compute OOD scores... (score: CSI)\n",
  1869. "One_class_real_mean: 0.6796440616169901\n",
  1870. "CNMC 2.2897 +- 0.8188 q0: -0.1330 q10: 1.2176 q20: 1.6025 q30: 1.8640 q40: 2.0833 q50: 2.2961 q60: 2.5135 q70: 2.7109 q80: 2.9502 q90: 3.3024 q100: 4.9905\n",
  1871. "one_class_0 1.8212 +- 0.6476 q0: -0.7580 q10: 1.0215 q20: 1.3188 q30: 1.4821 q40: 1.6494 q50: 1.7908 q60: 1.9457 q70: 2.1147 q80: 2.3124 q90: 2.6421 q100: 4.3585\n",
  1872. "[one_class_0 CSI 0.6796] [one_class_0 best 0.6796] \n",
  1873. "[one_class_mean CSI 0.6796] [one_class_mean best 0.6796] \n",
  1874. "0.6796\t0.6796\n"
  1875. ]
  1876. }
  1877. ],
  1878. "source": [
  1879. "# EVALUATION\n",
  1880. "# dataset : CNMC\n",
  1881. "# res : 450px\n",
  1882. "# id_class : hem\n",
  1883. "# epoch : 100\n",
  1884. "# shift_tr : blur\n",
  1885. "# crop : 0.08\n",
  1886. "# blur_sigma : 1\n",
  1887. "# color_dist : 0.5\n",
  1888. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 1 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma1.0_one_class_1/last.model\""
  1889. ]
  1890. },
  1891. {
  1892. "cell_type": "markdown",
  1893. "id": "9fd03e0e",
  1894. "metadata": {},
  1895. "source": [
  1896. "## other transformations"
  1897. ]
  1898. },
  1899. {
  1900. "cell_type": "code",
  1901. "execution_count": 151,
  1902. "id": "beda234d",
  1903. "metadata": {},
  1904. "outputs": [
  1905. {
  1906. "name": "stdout",
  1907. "output_type": "stream",
  1908. "text": [
  1909. "Pre-compute global statistics...\n",
  1910. "axis size: 3581 3581 3581 3581\n",
  1911. "weight_sim:\t0.0022\t0.0048\t0.0029\t0.0028\n",
  1912. "weight_shi:\t-3.2909\t-2.8657\t12.5482\t8.7034\n",
  1913. "Pre-compute features...\n",
  1914. "Compute OOD scores... (score: CSI)\n",
  1915. "One_class_real_mean: 0.5285322922047013\n",
  1916. "CNMC 2.0914 +- 0.5227 q0: 0.3459 q10: 1.3730 q20: 1.6394 q30: 1.8216 q40: 1.9820 q50: 2.1814 q60: 2.2788 q70: 2.3999 q80: 2.5247 q90: 2.7229 q100: 3.6842\n",
  1917. "one_class_0 2.0687 +- 0.3844 q0: 0.9708 q10: 1.5495 q20: 1.7411 q30: 1.8523 q40: 1.9633 q50: 2.0755 q60: 2.1823 q70: 2.2948 q80: 2.4057 q90: 2.5591 q100: 3.3615\n",
  1918. "[one_class_0 CSI 0.5285] [one_class_0 best 0.5285] \n",
  1919. "[one_class_mean CSI 0.5285] [one_class_mean best 0.5285] \n",
  1920. "0.5285\t0.5285\n"
  1921. ]
  1922. }
  1923. ],
  1924. "source": [
  1925. "# EVALUATION\n",
  1926. "# dataset : CNMC\n",
  1927. "# res : 450px\n",
  1928. "# id_class : hem\n",
  1929. "# epoch : 100\n",
  1930. "# shift_tr : rotation\n",
  1931. "# crop : 0.08\n",
  1932. "# color_dist : 0.5\n",
  1933. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type rotation --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_rotation_resize_factor0.08_color_dist0.5_one_class_1/last.model\""
  1934. ]
  1935. },
  1936. {
  1937. "cell_type": "code",
  1938. "execution_count": 152,
  1939. "id": "025aedc5",
  1940. "metadata": {
  1941. "scrolled": true
  1942. },
  1943. "outputs": [
  1944. {
  1945. "name": "stdout",
  1946. "output_type": "stream",
  1947. "text": [
  1948. "Pre-compute global statistics...\n",
  1949. "axis size: 3581 3581 3581 3581\n",
  1950. "weight_sim:\t0.0030\t0.0034\t0.0036\t0.0040\n",
  1951. "weight_shi:\t-0.0433\t0.5499\t-0.7289\t0.1057\n",
  1952. "Pre-compute features...\n",
  1953. "Compute OOD scores... (score: CSI)\n",
  1954. "One_class_real_mean: 0.4115717953392276\n",
  1955. "CNMC 2.0156 +- 0.1922 q0: 1.5401 q10: 1.7668 q20: 1.8500 q30: 1.9196 q40: 1.9652 q50: 2.0074 q60: 2.0619 q70: 2.1144 q80: 2.1791 q90: 2.2713 q100: 2.6170\n",
  1956. "one_class_0 2.0726 +- 0.1608 q0: 1.6384 q10: 1.8757 q20: 1.9493 q30: 1.9876 q40: 2.0244 q50: 2.0556 q60: 2.0929 q70: 2.1446 q80: 2.2026 q90: 2.3007 q100: 2.7631\n",
  1957. "[one_class_0 CSI 0.4116] [one_class_0 best 0.4116] \n",
  1958. "[one_class_mean CSI 0.4116] [one_class_mean best 0.4116] \n",
  1959. "0.4116\t0.4116\n"
  1960. ]
  1961. }
  1962. ],
  1963. "source": [
  1964. "# EVALUATION\n",
  1965. "# dataset : CNMC\n",
  1966. "# res : 450px\n",
  1967. "# id_class : hem\n",
  1968. "# epoch : 100\n",
  1969. "# shift_tr : cutperm\n",
  1970. "# crop : 0.08\n",
  1971. "# color_dist : 0.5\n",
  1972. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type cutperm --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 1 --load_path \"logs/id_hem/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_cutperm_resize_factor0.08_color_dist0.5_one_class_1/last.model\""
  1973. ]
  1974. },
  1975. {
  1976. "cell_type": "markdown",
  1977. "id": "7def804c",
  1978. "metadata": {},
  1979. "source": [
  1980. "# In-Distribution = ALL"
  1981. ]
  1982. },
  1983. {
  1984. "cell_type": "markdown",
  1985. "id": "4d826eb3",
  1986. "metadata": {},
  1987. "source": [
  1988. "# Combined shiftings"
  1989. ]
  1990. },
  1991. {
  1992. "cell_type": "code",
  1993. "execution_count": 153,
  1994. "id": "ed1501cd",
  1995. "metadata": {
  1996. "scrolled": true
  1997. },
  1998. "outputs": [
  1999. {
  2000. "name": "stdout",
  2001. "output_type": "stream",
  2002. "text": [
  2003. "Pre-compute global statistics...\n",
  2004. "axis size: 3527 3527 3527 3527\n",
  2005. "weight_sim:\t0.0099\t0.0102\t0.0065\t0.0102\n",
  2006. "weight_shi:\t-0.2651\t0.3988\t-0.4352\t-0.9217\n",
  2007. "Pre-compute features...\n",
  2008. "Compute OOD scores... (score: CSI)\n",
  2009. "One_class_real_mean: 0.39018776775134445\n",
  2010. "CNMC 1.9696 +- 0.3172 q0: 1.0059 q10: 1.5824 q20: 1.7007 q30: 1.7852 q40: 1.8767 q50: 1.9557 q60: 2.0335 q70: 2.1270 q80: 2.2281 q90: 2.3851 q100: 3.0697\n",
  2011. "one_class_1 2.1059 +- 0.3583 q0: 1.1658 q10: 1.6595 q20: 1.8070 q30: 1.9152 q40: 2.0053 q50: 2.0849 q60: 2.1740 q70: 2.2650 q80: 2.3979 q90: 2.5818 q100: 3.4632\n",
  2012. "[one_class_1 CSI 0.3902] [one_class_1 best 0.3902] \n",
  2013. "[one_class_mean CSI 0.3902] [one_class_mean best 0.3902] \n",
  2014. "0.3902\t0.3902\n"
  2015. ]
  2016. }
  2017. ],
  2018. "source": [
  2019. "# EVALUATION\n",
  2020. "# dataset : CNMC\n",
  2021. "# res : 450px\n",
  2022. "# id_class : all\n",
  2023. "# epoch : 100\n",
  2024. "# shift_tr : blur_randpers\n",
  2025. "# crop : 0.08\n",
  2026. "# blur_sigma : 2\n",
  2027. "# randpers : 0.75\n",
  2028. "# color_dist : 0.5\n",
  2029. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --distortion_scale 0.75 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur_randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_randpers_resize_factor0.08_color_dist0.5_one_class_0/last.model\""
  2030. ]
  2031. },
  2032. {
  2033. "cell_type": "code",
  2034. "execution_count": 154,
  2035. "id": "b471436b",
  2036. "metadata": {
  2037. "scrolled": true
  2038. },
  2039. "outputs": [
  2040. {
  2041. "name": "stdout",
  2042. "output_type": "stream",
  2043. "text": [
  2044. "Pre-compute global statistics...\n",
  2045. "axis size: 3527 3527 3527 3527\n",
  2046. "weight_sim:\t0.0067\t0.0129\t0.0065\t0.0086\n",
  2047. "weight_shi:\t-0.0850\t0.2249\t0.1729\t0.1702\n",
  2048. "Pre-compute features...\n",
  2049. "Compute OOD scores... (score: CSI)\n",
  2050. "One_class_real_mean: 0.4909736780805963\n",
  2051. "CNMC 2.1838 +- 0.3670 q0: 0.9506 q10: 1.6677 q20: 1.8649 q30: 2.0015 q40: 2.1242 q50: 2.2193 q60: 2.3160 q70: 2.4125 q80: 2.5084 q90: 2.6376 q100: 3.1795\n",
  2052. "one_class_1 2.1670 +- 0.4888 q0: 0.7892 q10: 1.4646 q20: 1.7498 q30: 1.9466 q40: 2.1070 q50: 2.2393 q60: 2.3641 q70: 2.4747 q80: 2.6032 q90: 2.7386 q100: 3.1321\n",
  2053. "[one_class_1 CSI 0.4910] [one_class_1 best 0.4910] \n",
  2054. "[one_class_mean CSI 0.4910] [one_class_mean best 0.4910] \n",
  2055. "0.4910\t0.4910\n"
  2056. ]
  2057. }
  2058. ],
  2059. "source": [
  2060. "# EVALUATION\n",
  2061. "# dataset : CNMC\n",
  2062. "# res : 450px\n",
  2063. "# id_class : all\n",
  2064. "# epoch : 100\n",
  2065. "# shift_tr : blur_sharp\n",
  2066. "# crop : 0.08\n",
  2067. "# blur_sigma : 2\n",
  2068. "# randpers : 0.75\n",
  2069. "# color_dist : 0.5\n",
  2070. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --sharpness_factor 5 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur_sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_sharp_resize_factor0.08_color_dist0.5_one_class_0/last.model\""
  2071. ]
  2072. },
  2073. {
  2074. "cell_type": "code",
  2075. "execution_count": 155,
  2076. "id": "5c08667d",
  2077. "metadata": {
  2078. "scrolled": true
  2079. },
  2080. "outputs": [
  2081. {
  2082. "name": "stdout",
  2083. "output_type": "stream",
  2084. "text": [
  2085. "Pre-compute global statistics...\n",
  2086. "axis size: 3527 3527 3527 3527\n",
  2087. "weight_sim:\t0.0058\t0.0071\t0.0060\t0.0060\n",
  2088. "weight_shi:\t-0.0229\t0.0795\t0.0649\t0.0666\n",
  2089. "Pre-compute features...\n",
  2090. "Compute OOD scores... (score: CSI)\n",
  2091. "One_class_real_mean: 0.45186742320663564\n",
  2092. "CNMC 2.0290 +- 0.0922 q0: 1.7841 q10: 1.9115 q20: 1.9486 q30: 1.9769 q40: 2.0011 q50: 2.0200 q60: 2.0479 q70: 2.0788 q80: 2.1081 q90: 2.1556 q100: 2.4408\n",
  2093. "one_class_1 2.0462 +- 0.1034 q0: 1.7679 q10: 1.9185 q20: 1.9546 q30: 1.9876 q40: 2.0171 q50: 2.0410 q60: 2.0679 q70: 2.0985 q80: 2.1328 q90: 2.1914 q100: 2.3683\n",
  2094. "[one_class_1 CSI 0.4519] [one_class_1 best 0.4519] \n",
  2095. "[one_class_mean CSI 0.4519] [one_class_mean best 0.4519] \n",
  2096. "0.4519\t0.4519\n"
  2097. ]
  2098. }
  2099. ],
  2100. "source": [
  2101. "# EVALUATION\n",
  2102. "# dataset : CNMC\n",
  2103. "# res : 450px\n",
  2104. "# id_class : all\n",
  2105. "# epoch : 100\n",
  2106. "# shift_tr : randpers_sharp\n",
  2107. "# crop : 0.08\n",
  2108. "# blur_sigma : 2\n",
  2109. "# randpers : 0.75\n",
  2110. "# color_dist : 0.5\n",
  2111. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --sharpness_factor 5 --distortion_scale 0.75 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers_sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_sharp_resize_factor0.08_color_dist0.5_one_class_0/last.model\""
  2112. ]
  2113. },
  2114. {
  2115. "cell_type": "code",
  2116. "execution_count": 156,
  2117. "id": "e1be886d",
  2118. "metadata": {
  2119. "scrolled": true
  2120. },
  2121. "outputs": [
  2122. {
  2123. "name": "stdout",
  2124. "output_type": "stream",
  2125. "text": [
  2126. "Pre-compute global statistics...\n",
  2127. "axis size: 3527 3527 3527 3527\n",
  2128. "weight_sim:\t0.0069\t0.0188\t0.0166\t0.0120\n",
  2129. "weight_shi:\t-0.1581\t0.1971\t0.2342\t0.3190\n",
  2130. "Pre-compute features...\n",
  2131. "Compute OOD scores... (score: CSI)\n",
  2132. "One_class_real_mean: 0.32064141322071316\n",
  2133. "CNMC 1.9454 +- 0.0810 q0: 1.7576 q10: 1.8630 q20: 1.8860 q30: 1.9012 q40: 1.9140 q50: 1.9316 q60: 1.9461 q70: 1.9640 q80: 1.9905 q90: 2.0476 q100: 2.4165\n",
  2134. "one_class_1 2.0265 +- 0.1592 q0: 1.7834 q10: 1.8887 q20: 1.9115 q30: 1.9346 q40: 1.9614 q50: 1.9884 q60: 2.0114 q70: 2.0559 q80: 2.1059 q90: 2.2091 q100: 3.1080\n",
  2135. "[one_class_1 CSI 0.3206] [one_class_1 best 0.3206] \n",
  2136. "[one_class_mean CSI 0.3206] [one_class_mean best 0.3206] \n",
  2137. "0.3206\t0.3206\n"
  2138. ]
  2139. }
  2140. ],
  2141. "source": [
  2142. "# EVALUATION\n",
  2143. "# dataset : CNMC\n",
  2144. "# res : 450px\n",
  2145. "# id_class : all\n",
  2146. "# epoch : 100\n",
  2147. "# shift_tr : blur_randpers_sharp\n",
  2148. "# crop : 0.08\n",
  2149. "# blur_sigma : 2\n",
  2150. "# randpers : 0.75\n",
  2151. "# color_dist : 0.5\n",
  2152. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --sharpness_factor 5 --distortion_scale 0.75 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur_randpers_sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_randpers_sharp_resize_factor0.08_color_dist0.5_one_class_0/last.model\""
  2153. ]
  2154. },
  2155. {
  2156. "cell_type": "markdown",
  2157. "id": "d8cd9c5a",
  2158. "metadata": {},
  2159. "source": [
  2160. "# Rotation"
  2161. ]
  2162. },
  2163. {
  2164. "cell_type": "code",
  2165. "execution_count": 157,
  2166. "id": "3f9748c5",
  2167. "metadata": {},
  2168. "outputs": [
  2169. {
  2170. "name": "stdout",
  2171. "output_type": "stream",
  2172. "text": [
  2173. "Pre-compute global statistics...\n",
  2174. "axis size: 3527 3527 3527 3527\n",
  2175. "weight_sim:\t0.0058\t0.0091\t0.0059\t0.0061\n",
  2176. "weight_shi:\t-20.2520\t5.6794\t4.4756\t-13.8486\n",
  2177. "Pre-compute features...\n",
  2178. "Compute OOD scores... (score: CSI)\n",
  2179. "One_class_real_mean: 0.6155293247855458\n",
  2180. "CNMC 2.2941 +- 0.4851 q0: 0.6171 q10: 1.6977 q20: 1.9170 q30: 2.0537 q40: 2.1467 q50: 2.2525 q60: 2.3680 q70: 2.5004 q80: 2.6952 q90: 2.9333 q100: 4.0615\n",
  2181. "one_class_1 2.0566 +- 0.6054 q0: 0.2141 q10: 1.2573 q20: 1.5313 q30: 1.7431 q40: 1.8966 q50: 2.0426 q60: 2.2221 q70: 2.3685 q80: 2.6045 q90: 2.8399 q100: 3.9073\n",
  2182. "[one_class_1 CSI 0.6155] [one_class_1 best 0.6155] \n",
  2183. "[one_class_mean CSI 0.6155] [one_class_mean best 0.6155] \n",
  2184. "0.6155\t0.6155\n"
  2185. ]
  2186. }
  2187. ],
  2188. "source": [
  2189. "###### EVALUATION\n",
  2190. "# dataset : CNMC\n",
  2191. "# res : 450px\n",
  2192. "# id_class : all\n",
  2193. "# epoch : 100\n",
  2194. "# shift_tr : rotation\n",
  2195. "# crop : 0.08\n",
  2196. "# color_dist : 0.5\n",
  2197. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type rotation --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_rotation_resize_factor0.08_color_dist0.5_one_class_0/last.model\""
  2198. ]
  2199. },
  2200. {
  2201. "cell_type": "markdown",
  2202. "id": "ed7a3ca6",
  2203. "metadata": {},
  2204. "source": [
  2205. "# Cutperm"
  2206. ]
  2207. },
  2208. {
  2209. "cell_type": "code",
  2210. "execution_count": 158,
  2211. "id": "47382eef",
  2212. "metadata": {},
  2213. "outputs": [
  2214. {
  2215. "name": "stdout",
  2216. "output_type": "stream",
  2217. "text": [
  2218. "Pre-compute global statistics...\n",
  2219. "axis size: 3527 3527 3527 3527\n",
  2220. "weight_sim:\t0.0033\t0.0040\t0.0048\t0.0059\n",
  2221. "weight_shi:\t-0.0422\t-0.2956\t0.3071\t0.0913\n",
  2222. "Pre-compute features...\n",
  2223. "Compute OOD scores... (score: CSI)\n",
  2224. "One_class_real_mean: 0.5637665967854647\n",
  2225. "CNMC 2.1340 +- 0.2713 q0: 1.5092 q10: 1.8054 q20: 1.8955 q30: 1.9635 q40: 2.0281 q50: 2.0900 q60: 2.1689 q70: 2.2729 q80: 2.3753 q90: 2.5306 q100: 2.8713\n",
  2226. "one_class_1 2.0681 +- 0.3216 q0: 1.3818 q10: 1.6678 q20: 1.7728 q30: 1.8582 q40: 1.9368 q50: 2.0391 q60: 2.1288 q70: 2.2616 q80: 2.3884 q90: 2.5307 q100: 2.8915\n",
  2227. "[one_class_1 CSI 0.5638] [one_class_1 best 0.5638] \n",
  2228. "[one_class_mean CSI 0.5638] [one_class_mean best 0.5638] \n",
  2229. "0.5638\t0.5638\n"
  2230. ]
  2231. }
  2232. ],
  2233. "source": [
  2234. "###### EVALUATION\n",
  2235. "# dataset : CNMC\n",
  2236. "# res : 450px\n",
  2237. "# id_class : all\n",
  2238. "# epoch : 100\n",
  2239. "# shift_tr : cutperm\n",
  2240. "# crop : 0.08\n",
  2241. "# color_dist : 0.5\n",
  2242. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type cutperm --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_cutperm_resize_factor0.08_color_dist0.5_one_class_0/last.model\""
  2243. ]
  2244. },
  2245. {
  2246. "cell_type": "markdown",
  2247. "id": "e338538b",
  2248. "metadata": {},
  2249. "source": [
  2250. "# Rotated Dataset 4"
  2251. ]
  2252. },
  2253. {
  2254. "cell_type": "code",
  2255. "execution_count": 69,
  2256. "id": "18aa1694",
  2257. "metadata": {},
  2258. "outputs": [
  2259. {
  2260. "name": "stdout",
  2261. "output_type": "stream",
  2262. "text": [
  2263. "Pre-compute global statistics...\n",
  2264. "axis size: 3527 3527 3527 3527\n",
  2265. "weight_sim:\t0.0089\t0.0071\t0.0082\t0.0060\n",
  2266. "weight_shi:\t-0.0826\t0.1155\t0.1144\t0.1138\n",
  2267. "Pre-compute features...\n",
  2268. "Compute OOD scores... (score: CSI)\n",
  2269. "One_class_real_mean: 0.6829627857280305\n",
  2270. "CNMC 2.0823 +- 0.1590 q0: 1.4861 q10: 1.8867 q20: 1.9512 q30: 1.9962 q40: 2.0434 q50: 2.0844 q60: 2.1284 q70: 2.1653 q80: 2.2150 q90: 2.2784 q100: 2.7066\n",
  2271. "one_class_1 1.9798 +- 0.1471 q0: 1.4589 q10: 1.7996 q20: 1.8601 q30: 1.9145 q40: 1.9503 q50: 1.9828 q60: 2.0164 q70: 2.0541 q80: 2.1007 q90: 2.1670 q100: 2.3931\n",
  2272. "[one_class_1 CSI 0.6830] [one_class_1 best 0.6830] \n",
  2273. "[one_class_mean CSI 0.6830] [one_class_mean best 0.6830] \n",
  2274. "0.6830\t0.6830\n"
  2275. ]
  2276. }
  2277. ],
  2278. "source": [
  2279. "###### EVALUATION\n",
  2280. "# dataset : CNMC_ROT4\n",
  2281. "# res : 450px\n",
  2282. "# id_class : all\n",
  2283. "# epoch : 100\n",
  2284. "# shift_tr : sharp\n",
  2285. "# crop : 0.08\n",
  2286. "# sharp : 64\n",
  2287. "# color_dist : 0.5\n",
  2288. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 64 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/dataset_rotated_4/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor64.0_one_class_0/last.model\""
  2289. ]
  2290. },
  2291. {
  2292. "cell_type": "code",
  2293. "execution_count": 70,
  2294. "id": "95e84b59",
  2295. "metadata": {},
  2296. "outputs": [
  2297. {
  2298. "name": "stdout",
  2299. "output_type": "stream",
  2300. "text": [
  2301. "Pre-compute global statistics...\n",
  2302. "axis size: 3527 3527 3527 3527\n",
  2303. "weight_sim:\t0.0076\t0.0081\t0.0080\t0.0086\n",
  2304. "weight_shi:\t-0.1382\t1.2588\t2.1567\t0.5287\n",
  2305. "Pre-compute features...\n",
  2306. "Compute OOD scores... (score: CSI)\n",
  2307. "One_class_real_mean: 0.3485907290938738\n",
  2308. "CNMC 1.8551 +- 0.4346 q0: 0.8263 q10: 1.3471 q20: 1.5032 q30: 1.6156 q40: 1.7119 q50: 1.8134 q60: 1.9097 q70: 2.0306 q80: 2.1600 q90: 2.4047 q100: 4.4743\n",
  2309. "one_class_1 2.1133 +- 0.5033 q0: 0.9826 q10: 1.5132 q20: 1.7004 q30: 1.8184 q40: 1.9124 q50: 2.0310 q60: 2.1594 q70: 2.3078 q80: 2.5394 q90: 2.7696 q100: 4.0888\n",
  2310. "[one_class_1 CSI 0.3486] [one_class_1 best 0.3486] \n",
  2311. "[one_class_mean CSI 0.3486] [one_class_mean best 0.3486] \n",
  2312. "0.3486\t0.3486\n"
  2313. ]
  2314. }
  2315. ],
  2316. "source": [
  2317. "###### EVALUATION\n",
  2318. "# dataset : CNMC_ROT4\n",
  2319. "# res : 450px\n",
  2320. "# id_class : all\n",
  2321. "# epoch : 100\n",
  2322. "# shift_tr : randpers\n",
  2323. "# crop : 0.08\n",
  2324. "# randpers : 0.75\n",
  2325. "# color_dist : 0.5\n",
  2326. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.75 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/dataset_rotated_4/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.75_one_class_0/last.model\""
  2327. ]
  2328. },
  2329. {
  2330. "cell_type": "code",
  2331. "execution_count": 159,
  2332. "id": "982cf5a4",
  2333. "metadata": {},
  2334. "outputs": [
  2335. {
  2336. "name": "stdout",
  2337. "output_type": "stream",
  2338. "text": [
  2339. "Pre-compute global statistics...\n",
  2340. "axis size: 3527 3527 3527 3527\n",
  2341. "weight_sim:\t0.0137\t0.0152\t0.0140\t0.0126\n",
  2342. "weight_shi:\t-0.1440\t0.3135\t0.4775\t0.4211\n",
  2343. "Pre-compute features...\n",
  2344. "Compute OOD scores... (score: CSI)\n",
  2345. "One_class_real_mean: 0.3167783246741409\n",
  2346. "CNMC 1.9366 +- 0.1773 q0: 1.4437 q10: 1.7285 q20: 1.7909 q30: 1.8431 q40: 1.8862 q50: 1.9192 q60: 1.9569 q70: 2.0073 q80: 2.0694 q90: 2.1765 q100: 2.5655\n",
  2347. "one_class_1 2.0674 +- 0.2071 q0: 1.6099 q10: 1.8331 q20: 1.8954 q30: 1.9420 q40: 1.9859 q50: 2.0320 q60: 2.0882 q70: 2.1432 q80: 2.2308 q90: 2.3727 q100: 2.8015\n",
  2348. "[one_class_1 CSI 0.3168] [one_class_1 best 0.3168] \n",
  2349. "[one_class_mean CSI 0.3168] [one_class_mean best 0.3168] \n",
  2350. "0.3168\t0.3168\n"
  2351. ]
  2352. }
  2353. ],
  2354. "source": [
  2355. "# EVALUATION\n",
  2356. "# dataset : CNMC_ROT4\n",
  2357. "# res : 450px\n",
  2358. "# id_class : all\n",
  2359. "# epoch : 100\n",
  2360. "# shift_tr : blur\n",
  2361. "# crop : 0.08\n",
  2362. "# blur_sigma : 2\n",
  2363. "# color_dist : 0.5\n",
  2364. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/dataset_rotated_4/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma2.0_one_class_0/last.model\""
  2365. ]
  2366. },
  2367. {
  2368. "cell_type": "markdown",
  2369. "id": "c9c8f555",
  2370. "metadata": {},
  2371. "source": [
  2372. "# Sharpness Factor"
  2373. ]
  2374. },
  2375. {
  2376. "cell_type": "code",
  2377. "execution_count": 72,
  2378. "id": "ac35a164",
  2379. "metadata": {},
  2380. "outputs": [
  2381. {
  2382. "name": "stdout",
  2383. "output_type": "stream",
  2384. "text": [
  2385. "Pre-compute global statistics...\n",
  2386. "axis size: 3527 3527 3527 3527\n",
  2387. "weight_sim:\t0.0050\t0.0011\t0.0010\t0.0009\n",
  2388. "weight_shi:\t-0.0433\t0.0726\t0.2303\t0.0769\n",
  2389. "Pre-compute features...\n",
  2390. "Compute OOD scores... (score: CSI)\n",
  2391. "One_class_real_mean: 0.5375965930382118\n",
  2392. "CNMC 1.9981 +- 0.1008 q0: 1.5969 q10: 1.8641 q20: 1.9163 q30: 1.9503 q40: 1.9776 q50: 2.0056 q60: 2.0329 q70: 2.0563 q80: 2.0814 q90: 2.1226 q100: 2.2687\n",
  2393. "one_class_1 1.9867 +- 0.1056 q0: 1.6492 q10: 1.8484 q20: 1.8943 q30: 1.9323 q40: 1.9603 q50: 1.9909 q60: 2.0131 q70: 2.0457 q80: 2.0783 q90: 2.1193 q100: 2.2764\n",
  2394. "[one_class_1 CSI 0.5376] [one_class_1 best 0.5376] \n",
  2395. "[one_class_mean CSI 0.5376] [one_class_mean best 0.5376] \n",
  2396. "0.5376\t0.5376\n"
  2397. ]
  2398. }
  2399. ],
  2400. "source": [
  2401. "###### EVALUATION\n",
  2402. "# dataset : CNMC\n",
  2403. "# res : 450px\n",
  2404. "# id_class : all\n",
  2405. "# epoch : 100\n",
  2406. "# shift_tr : sharp\n",
  2407. "# crop : 0.08\n",
  2408. "# sharp : 4096\n",
  2409. "# color_dist : 0.5\n",
  2410. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 4096 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor4096.0_one_class_0/last.model\""
  2411. ]
  2412. },
  2413. {
  2414. "cell_type": "code",
  2415. "execution_count": 73,
  2416. "id": "49250ae3",
  2417. "metadata": {},
  2418. "outputs": [
  2419. {
  2420. "name": "stdout",
  2421. "output_type": "stream",
  2422. "text": [
  2423. "Pre-compute global statistics...\n",
  2424. "axis size: 3527 3527 3527 3527\n",
  2425. "weight_sim:\t0.0099\t0.0038\t0.0037\t0.0035\n",
  2426. "weight_shi:\t-0.0601\t0.0628\t0.0572\t0.0620\n",
  2427. "Pre-compute features...\n",
  2428. "Compute OOD scores... (score: CSI)\n",
  2429. "One_class_real_mean: 0.5004063743809436\n",
  2430. "CNMC 2.1367 +- 0.2510 q0: 1.4456 q10: 1.8346 q20: 1.9233 q30: 1.9922 q40: 2.0485 q50: 2.1189 q60: 2.1860 q70: 2.2597 q80: 2.3425 q90: 2.4668 q100: 3.2275\n",
  2431. "one_class_1 2.1346 +- 0.3755 q0: 1.1971 q10: 1.6290 q20: 1.8287 q30: 1.9361 q40: 2.0319 q50: 2.1191 q60: 2.2264 q70: 2.3246 q80: 2.4509 q90: 2.6481 q100: 3.1412\n",
  2432. "[one_class_1 CSI 0.5004] [one_class_1 best 0.5004] \n",
  2433. "[one_class_mean CSI 0.5004] [one_class_mean best 0.5004] \n",
  2434. "0.5004\t0.5004\n"
  2435. ]
  2436. }
  2437. ],
  2438. "source": [
  2439. "###### EVALUATION\n",
  2440. "# dataset : CNMC\n",
  2441. "# res : 450px\n",
  2442. "# id_class : all\n",
  2443. "# epoch : 100\n",
  2444. "# shift_tr : sharp\n",
  2445. "# crop : 0.08\n",
  2446. "# sharp : 2048\n",
  2447. "# color_dist : 0.5\n",
  2448. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 2048 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor2048.0_one_class_0/last.model\""
  2449. ]
  2450. },
  2451. {
  2452. "cell_type": "code",
  2453. "execution_count": 74,
  2454. "id": "0bd84a7e",
  2455. "metadata": {},
  2456. "outputs": [
  2457. {
  2458. "name": "stdout",
  2459. "output_type": "stream",
  2460. "text": [
  2461. "Pre-compute global statistics...\n",
  2462. "axis size: 3527 3527 3527 3527\n",
  2463. "weight_sim:\t0.0242\t0.0050\t0.0046\t0.0044\n",
  2464. "weight_shi:\t-0.0828\t0.0645\t0.0669\t0.0596\n",
  2465. "Pre-compute features...\n",
  2466. "Compute OOD scores... (score: CSI)\n",
  2467. "One_class_real_mean: 0.521132733772876\n",
  2468. "CNMC 2.1385 +- 0.2030 q0: 1.5179 q10: 1.8773 q20: 1.9648 q30: 2.0325 q40: 2.0825 q50: 2.1336 q60: 2.1808 q70: 2.2389 q80: 2.3102 q90: 2.3997 q100: 2.7902\n",
  2469. "one_class_1 2.1145 +- 0.2767 q0: 1.3283 q10: 1.7725 q20: 1.8865 q30: 1.9760 q40: 2.0428 q50: 2.1166 q60: 2.1976 q70: 2.2709 q80: 2.3529 q90: 2.4697 q100: 2.8054\n",
  2470. "[one_class_1 CSI 0.5211] [one_class_1 best 0.5211] \n",
  2471. "[one_class_mean CSI 0.5211] [one_class_mean best 0.5211] \n",
  2472. "0.5211\t0.5211\n"
  2473. ]
  2474. }
  2475. ],
  2476. "source": [
  2477. "###### EVALUATION\n",
  2478. "# dataset : CNMC\n",
  2479. "# res : 450px\n",
  2480. "# id_class : all\n",
  2481. "# epoch : 100\n",
  2482. "# shift_tr : sharp\n",
  2483. "# crop : 0.08\n",
  2484. "# sharp : 1024\n",
  2485. "# color_dist : 0.5\n",
  2486. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 1024 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor1024.0_one_class_0/last.model\""
  2487. ]
  2488. },
  2489. {
  2490. "cell_type": "code",
  2491. "execution_count": 75,
  2492. "id": "7084a03f",
  2493. "metadata": {},
  2494. "outputs": [
  2495. {
  2496. "name": "stdout",
  2497. "output_type": "stream",
  2498. "text": [
  2499. "Pre-compute global statistics...\n",
  2500. "axis size: 3527 3527 3527 3527\n",
  2501. "weight_sim:\t0.0059\t0.0055\t0.0051\t0.0051\n",
  2502. "weight_shi:\t-0.0132\t0.0371\t0.0377\t0.0376\n",
  2503. "Pre-compute features...\n",
  2504. "Compute OOD scores... (score: CSI)\n",
  2505. "One_class_real_mean: 0.5495593179999798\n",
  2506. "CNMC 2.0729 +- 0.0917 q0: 1.7983 q10: 1.9554 q20: 1.9947 q30: 2.0222 q40: 2.0461 q50: 2.0659 q60: 2.0930 q70: 2.1220 q80: 2.1540 q90: 2.1973 q100: 2.3686\n",
  2507. "one_class_1 2.0536 +- 0.1203 q0: 1.7288 q10: 1.9078 q20: 1.9506 q30: 1.9885 q40: 2.0186 q50: 2.0481 q60: 2.0860 q70: 2.1206 q80: 2.1548 q90: 2.2130 q100: 2.3754\n",
  2508. "[one_class_1 CSI 0.5496] [one_class_1 best 0.5496] \n",
  2509. "[one_class_mean CSI 0.5496] [one_class_mean best 0.5496] \n",
  2510. "0.5496\t0.5496\n"
  2511. ]
  2512. }
  2513. ],
  2514. "source": [
  2515. "###### EVALUATION\n",
  2516. "# dataset : CNMC\n",
  2517. "# res : 450px\n",
  2518. "# id_class : all\n",
  2519. "# epoch : 100\n",
  2520. "# shift_tr : sharp\n",
  2521. "# crop : 0.08\n",
  2522. "# sharp : 512\n",
  2523. "# color_dist : 0.5\n",
  2524. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 512 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor512.0_one_class_0/last.model\""
  2525. ]
  2526. },
  2527. {
  2528. "cell_type": "code",
  2529. "execution_count": 76,
  2530. "id": "7609406d",
  2531. "metadata": {},
  2532. "outputs": [
  2533. {
  2534. "name": "stdout",
  2535. "output_type": "stream",
  2536. "text": [
  2537. "Pre-compute global statistics...\n",
  2538. "axis size: 3527 3527 3527 3527\n",
  2539. "weight_sim:\t0.0033\t0.0016\t0.0015\t0.0015\n",
  2540. "weight_shi:\t-0.0626\t0.0548\t0.0482\t0.0476\n",
  2541. "Pre-compute features...\n",
  2542. "Compute OOD scores... (score: CSI)\n",
  2543. "One_class_real_mean: 0.5079350611207324\n",
  2544. "CNMC 2.1463 +- 0.2056 q0: 1.5154 q10: 1.8902 q20: 1.9661 q30: 2.0278 q40: 2.0864 q50: 2.1497 q60: 2.1995 q70: 2.2608 q80: 2.3272 q90: 2.4144 q100: 2.8480\n",
  2545. "one_class_1 2.1363 +- 0.2821 q0: 1.3866 q10: 1.7738 q20: 1.8962 q30: 1.9787 q40: 2.0632 q50: 2.1372 q60: 2.2119 q70: 2.2966 q80: 2.3999 q90: 2.5028 q100: 2.7673\n",
  2546. "[one_class_1 CSI 0.5079] [one_class_1 best 0.5079] \n",
  2547. "[one_class_mean CSI 0.5079] [one_class_mean best 0.5079] \n",
  2548. "0.5079\t0.5079\n"
  2549. ]
  2550. }
  2551. ],
  2552. "source": [
  2553. "###### EVALUATION\n",
  2554. "# dataset : CNMC\n",
  2555. "# res : 450px\n",
  2556. "# id_class : all\n",
  2557. "# epoch : 100\n",
  2558. "# shift_tr : sharp\n",
  2559. "# crop : 0.08\n",
  2560. "# sharp : 256\n",
  2561. "# color_dist : 0.5\n",
  2562. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 256 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor256.0_one_class_0/last.model\""
  2563. ]
  2564. },
  2565. {
  2566. "cell_type": "code",
  2567. "execution_count": 77,
  2568. "id": "aad2a734",
  2569. "metadata": {},
  2570. "outputs": [
  2571. {
  2572. "name": "stdout",
  2573. "output_type": "stream",
  2574. "text": [
  2575. "Pre-compute global statistics...\n",
  2576. "axis size: 3527 3527 3527 3527\n",
  2577. "weight_sim:\t0.0048\t0.0080\t0.0087\t0.0057\n",
  2578. "weight_shi:\t-0.0840\t0.0954\t0.0919\t0.0779\n",
  2579. "Pre-compute features...\n",
  2580. "Compute OOD scores... (score: CSI)\n",
  2581. "One_class_real_mean: 0.5942041645145282\n",
  2582. "CNMC 2.0494 +- 0.1428 q0: 1.7036 q10: 1.8748 q20: 1.9344 q30: 1.9773 q40: 2.0081 q50: 2.0436 q60: 2.0718 q70: 2.1122 q80: 2.1561 q90: 2.2278 q100: 2.7563\n",
  2583. "one_class_1 2.0001 +- 0.1729 q0: 1.5526 q10: 1.7899 q20: 1.8590 q30: 1.9109 q40: 1.9589 q50: 1.9977 q60: 2.0327 q70: 2.0653 q80: 2.1332 q90: 2.2229 q100: 2.6275\n",
  2584. "[one_class_1 CSI 0.5942] [one_class_1 best 0.5942] \n",
  2585. "[one_class_mean CSI 0.5942] [one_class_mean best 0.5942] \n",
  2586. "0.5942\t0.5942\n"
  2587. ]
  2588. }
  2589. ],
  2590. "source": [
  2591. "###### EVALUATION\n",
  2592. "# dataset : CNMC\n",
  2593. "# res : 450px\n",
  2594. "# id_class : all\n",
  2595. "# epoch : 100\n",
  2596. "# shift_tr : sharp\n",
  2597. "# crop : 0.08\n",
  2598. "# sharp : 128\n",
  2599. "# color_dist : 0.5\n",
  2600. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 128 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor128.0_one_class_0/last.model\""
  2601. ]
  2602. },
  2603. {
  2604. "cell_type": "code",
  2605. "execution_count": 78,
  2606. "id": "eceb0082",
  2607. "metadata": {},
  2608. "outputs": [
  2609. {
  2610. "name": "stdout",
  2611. "output_type": "stream",
  2612. "text": [
  2613. "Pre-compute global statistics...\n",
  2614. "axis size: 3527 3527 3527 3527\n",
  2615. "weight_sim:\t0.0055\t0.0037\t0.0058\t0.0037\n",
  2616. "weight_shi:\t-0.1448\t0.1735\t0.1588\t0.1423\n",
  2617. "Pre-compute features...\n",
  2618. "Compute OOD scores... (score: CSI)\n",
  2619. "One_class_real_mean: 0.7104164767720962\n",
  2620. "CNMC 2.0883 +- 0.1055 q0: 1.7978 q10: 1.9497 q20: 1.9997 q30: 2.0352 q40: 2.0642 q50: 2.0925 q60: 2.1162 q70: 2.1440 q80: 2.1764 q90: 2.2198 q100: 2.4996\n",
  2621. "one_class_1 1.9981 +- 0.1273 q0: 1.6099 q10: 1.8403 q20: 1.8878 q30: 1.9263 q40: 1.9618 q50: 1.9937 q60: 2.0157 q70: 2.0606 q80: 2.1110 q90: 2.1738 q100: 2.4795\n",
  2622. "[one_class_1 CSI 0.7104] [one_class_1 best 0.7104] \n",
  2623. "[one_class_mean CSI 0.7104] [one_class_mean best 0.7104] \n",
  2624. "0.7104\t0.7104\n"
  2625. ]
  2626. }
  2627. ],
  2628. "source": [
  2629. "###### EVALUATION\n",
  2630. "# dataset : CNMC\n",
  2631. "# res : 450px\n",
  2632. "# id_class : all\n",
  2633. "# epoch : 100\n",
  2634. "# shift_tr : sharp\n",
  2635. "# crop : 0.08\n",
  2636. "# sharp : 64\n",
  2637. "# color_dist : 0.5\n",
  2638. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 64 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor64.0_one_class_0/last.model\""
  2639. ]
  2640. },
  2641. {
  2642. "cell_type": "code",
  2643. "execution_count": 79,
  2644. "id": "7c881700",
  2645. "metadata": {},
  2646. "outputs": [
  2647. {
  2648. "name": "stdout",
  2649. "output_type": "stream",
  2650. "text": [
  2651. "Pre-compute global statistics...\n",
  2652. "axis size: 3527 3527 3527 3527\n",
  2653. "weight_sim:\t0.0028\t0.0085\t0.0044\t0.0097\n",
  2654. "weight_shi:\t-0.0235\t0.0638\t0.0549\t0.0541\n",
  2655. "Pre-compute features...\n",
  2656. "Compute OOD scores... (score: CSI)\n",
  2657. "One_class_real_mean: 0.5937648750746918\n",
  2658. "CNMC 2.0002 +- 0.1702 q0: 1.6494 q10: 1.8157 q20: 1.8583 q30: 1.8929 q40: 1.9291 q50: 1.9695 q60: 2.0094 q70: 2.0576 q80: 2.1345 q90: 2.2419 q100: 2.8225\n",
  2659. "one_class_1 1.9446 +- 0.1597 q0: 1.5613 q10: 1.7697 q20: 1.8122 q30: 1.8531 q40: 1.8816 q50: 1.9188 q60: 1.9583 q70: 2.0091 q80: 2.0737 q90: 2.1568 q100: 2.5480\n",
  2660. "[one_class_1 CSI 0.5938] [one_class_1 best 0.5938] \n",
  2661. "[one_class_mean CSI 0.5938] [one_class_mean best 0.5938] \n",
  2662. "0.5938\t0.5938\n"
  2663. ]
  2664. }
  2665. ],
  2666. "source": [
  2667. "###### EVALUATION\n",
  2668. "# dataset : CNMC\n",
  2669. "# res : 450px\n",
  2670. "# id_class : all\n",
  2671. "# epoch : 100\n",
  2672. "# shift_tr : sharp\n",
  2673. "# crop : 0.08\n",
  2674. "# sharp : 32\n",
  2675. "# color_dist : 0.5\n",
  2676. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 32 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor32.0_one_class_0/last.model\""
  2677. ]
  2678. },
  2679. {
  2680. "cell_type": "code",
  2681. "execution_count": 80,
  2682. "id": "afaa2706",
  2683. "metadata": {},
  2684. "outputs": [
  2685. {
  2686. "name": "stdout",
  2687. "output_type": "stream",
  2688. "text": [
  2689. "Pre-compute global statistics...\n",
  2690. "axis size: 3527 3527 3527 3527\n",
  2691. "weight_sim:\t0.0030\t0.0048\t0.0042\t0.0054\n",
  2692. "weight_shi:\t-0.0352\t0.0883\t0.0761\t0.0693\n",
  2693. "Pre-compute features...\n",
  2694. "Compute OOD scores... (score: CSI)\n",
  2695. "One_class_real_mean: 0.5747906095868907\n",
  2696. "CNMC 2.0814 +- 0.1200 q0: 1.7198 q10: 1.9284 q20: 1.9790 q30: 2.0137 q40: 2.0462 q50: 2.0794 q60: 2.1084 q70: 2.1462 q80: 2.1929 q90: 2.2480 q100: 2.3614\n",
  2697. "one_class_1 2.0319 +- 0.1736 q0: 1.4826 q10: 1.8007 q20: 1.8786 q30: 1.9545 q40: 1.9968 q50: 2.0343 q60: 2.0956 q70: 2.1409 q80: 2.1917 q90: 2.2512 q100: 2.3875\n",
  2698. "[one_class_1 CSI 0.5748] [one_class_1 best 0.5748] \n",
  2699. "[one_class_mean CSI 0.5748] [one_class_mean best 0.5748] \n",
  2700. "0.5748\t0.5748\n"
  2701. ]
  2702. }
  2703. ],
  2704. "source": [
  2705. "###### EVALUATION\n",
  2706. "# dataset : CNMC\n",
  2707. "# res : 450px\n",
  2708. "# id_class : all\n",
  2709. "# epoch : 100\n",
  2710. "# shift_tr : sharp\n",
  2711. "# crop : 0.08\n",
  2712. "# sharp : 16\n",
  2713. "# color_dist : 0.5\n",
  2714. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 16 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor16.0_one_class_0/last.model\""
  2715. ]
  2716. },
  2717. {
  2718. "cell_type": "code",
  2719. "execution_count": 81,
  2720. "id": "374eec9c",
  2721. "metadata": {},
  2722. "outputs": [
  2723. {
  2724. "name": "stdout",
  2725. "output_type": "stream",
  2726. "text": [
  2727. "Pre-compute global statistics...\n",
  2728. "axis size: 3527 3527 3527 3527\n",
  2729. "weight_sim:\t0.0040\t0.0039\t0.0044\t0.0039\n",
  2730. "weight_shi:\t-0.0360\t0.1191\t0.0847\t0.0773\n",
  2731. "Pre-compute features...\n",
  2732. "Compute OOD scores... (score: CSI)\n",
  2733. "One_class_real_mean: 0.47723417292052783\n",
  2734. "CNMC 2.1169 +- 0.2902 q0: 1.3060 q10: 1.7390 q20: 1.8614 q30: 1.9501 q40: 2.0312 q50: 2.1085 q60: 2.1828 q70: 2.2782 q80: 2.3917 q90: 2.5189 q100: 3.0332\n",
  2735. "one_class_1 2.1411 +- 0.3676 q0: 1.2368 q10: 1.6509 q20: 1.8257 q30: 1.9349 q40: 2.0555 q50: 2.1498 q60: 2.2467 q70: 2.3477 q80: 2.4742 q90: 2.6155 q100: 3.2105\n",
  2736. "[one_class_1 CSI 0.4772] [one_class_1 best 0.4772] \n",
  2737. "[one_class_mean CSI 0.4772] [one_class_mean best 0.4772] \n",
  2738. "0.4772\t0.4772\n"
  2739. ]
  2740. }
  2741. ],
  2742. "source": [
  2743. "###### EVALUATION\n",
  2744. "# dataset : CNMC\n",
  2745. "# res : 450px\n",
  2746. "# id_class : all\n",
  2747. "# epoch : 100\n",
  2748. "# shift_tr : sharp\n",
  2749. "# crop : 0.08\n",
  2750. "# sharp : 8\n",
  2751. "# color_dist : 0.5\n",
  2752. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 8 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor8.0_one_class_0/last.model\""
  2753. ]
  2754. },
  2755. {
  2756. "cell_type": "code",
  2757. "execution_count": 82,
  2758. "id": "2b907319",
  2759. "metadata": {},
  2760. "outputs": [
  2761. {
  2762. "name": "stdout",
  2763. "output_type": "stream",
  2764. "text": [
  2765. "Pre-compute global statistics...\n",
  2766. "axis size: 3527 3527 3527 3527\n",
  2767. "weight_sim:\t0.0120\t0.0137\t0.0113\t0.0151\n",
  2768. "weight_shi:\t-0.0230\t0.0744\t0.0702\t0.0797\n",
  2769. "Pre-compute features...\n",
  2770. "Compute OOD scores... (score: CSI)\n",
  2771. "One_class_real_mean: 0.7440512360870578\n",
  2772. "CNMC 2.0897 +- 0.1280 q0: 1.6279 q10: 1.9253 q20: 1.9937 q30: 2.0344 q40: 2.0646 q50: 2.0929 q60: 2.1235 q70: 2.1514 q80: 2.1925 q90: 2.2452 q100: 2.5109\n",
  2773. "one_class_1 1.9564 +- 0.1648 q0: 1.2763 q10: 1.7402 q20: 1.8197 q30: 1.8821 q40: 1.9240 q50: 1.9604 q60: 2.0013 q70: 2.0434 q80: 2.0908 q90: 2.1604 q100: 2.4535\n",
  2774. "[one_class_1 CSI 0.7441] [one_class_1 best 0.7441] \n",
  2775. "[one_class_mean CSI 0.7441] [one_class_mean best 0.7441] \n",
  2776. "0.7441\t0.7441\n"
  2777. ]
  2778. }
  2779. ],
  2780. "source": [
  2781. "###### EVALUATION\n",
  2782. "# dataset : CNMC\n",
  2783. "# res : 450px\n",
  2784. "# id_class : all\n",
  2785. "# epoch : 100\n",
  2786. "# shift_tr : sharp\n",
  2787. "# crop : 0.08\n",
  2788. "# sharp : 5\n",
  2789. "# color_dist : 0.5\n",
  2790. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 5 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor5.0_one_class_0/last.model\""
  2791. ]
  2792. },
  2793. {
  2794. "cell_type": "code",
  2795. "execution_count": 83,
  2796. "id": "eadc9f63",
  2797. "metadata": {},
  2798. "outputs": [
  2799. {
  2800. "name": "stdout",
  2801. "output_type": "stream",
  2802. "text": [
  2803. "Pre-compute global statistics...\n",
  2804. "axis size: 3527 3527 3527 3527\n",
  2805. "weight_sim:\t0.0044\t0.0050\t0.0036\t0.0045\n",
  2806. "weight_shi:\t-0.0174\t0.0520\t0.0435\t0.0457\n",
  2807. "Pre-compute features...\n",
  2808. "Compute OOD scores... (score: CSI)\n",
  2809. "One_class_real_mean: 0.6856960015799229\n",
  2810. "CNMC 2.0337 +- 0.0637 q0: 1.7790 q10: 1.9526 q20: 1.9802 q30: 2.0010 q40: 2.0182 q50: 2.0348 q60: 2.0490 q70: 2.0674 q80: 2.0882 q90: 2.1155 q100: 2.2475\n",
  2811. "one_class_1 1.9842 +- 0.0803 q0: 1.6728 q10: 1.8819 q20: 1.9195 q30: 1.9483 q40: 1.9679 q50: 1.9875 q60: 2.0072 q70: 2.0236 q80: 2.0479 q90: 2.0840 q100: 2.2335\n",
  2812. "[one_class_1 CSI 0.6857] [one_class_1 best 0.6857] \n",
  2813. "[one_class_mean CSI 0.6857] [one_class_mean best 0.6857] \n",
  2814. "0.6857\t0.6857\n"
  2815. ]
  2816. }
  2817. ],
  2818. "source": [
  2819. "###### EVALUATION\n",
  2820. "# dataset : CNMC\n",
  2821. "# res : 450px\n",
  2822. "# id_class : all\n",
  2823. "# epoch : 100\n",
  2824. "# shift_tr : sharp\n",
  2825. "# crop : 0.08\n",
  2826. "# sharp : 4\n",
  2827. "# color_dist : 0.5\n",
  2828. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 4 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor4.0_one_class_0/last.model\""
  2829. ]
  2830. },
  2831. {
  2832. "cell_type": "code",
  2833. "execution_count": 84,
  2834. "id": "66a30bac",
  2835. "metadata": {},
  2836. "outputs": [
  2837. {
  2838. "name": "stdout",
  2839. "output_type": "stream",
  2840. "text": [
  2841. "Pre-compute global statistics...\n",
  2842. "axis size: 3527 3527 3527 3527\n",
  2843. "weight_sim:\t0.0096\t0.0088\t0.0090\t0.0096\n",
  2844. "weight_shi:\t-0.0320\t0.1007\t0.1076\t0.0998\n",
  2845. "Pre-compute features...\n",
  2846. "Compute OOD scores... (score: CSI)\n",
  2847. "One_class_real_mean: 0.49014067389785193\n",
  2848. "CNMC 2.0877 +- 0.2810 q0: 1.3173 q10: 1.7265 q20: 1.8460 q30: 1.9306 q40: 2.0048 q50: 2.0768 q60: 2.1398 q70: 2.2252 q80: 2.3252 q90: 2.4627 q100: 3.0835\n",
  2849. "one_class_1 2.0957 +- 0.3295 q0: 1.1248 q10: 1.6700 q20: 1.8185 q30: 1.9197 q40: 2.0144 q50: 2.0849 q60: 2.1813 q70: 2.2611 q80: 2.3718 q90: 2.5219 q100: 3.0920\n",
  2850. "[one_class_1 CSI 0.4901] [one_class_1 best 0.4901] \n",
  2851. "[one_class_mean CSI 0.4901] [one_class_mean best 0.4901] \n",
  2852. "0.4901\t0.4901\n"
  2853. ]
  2854. }
  2855. ],
  2856. "source": [
  2857. "###### EVALUATION\n",
  2858. "# dataset : CNMC\n",
  2859. "# res : 450px\n",
  2860. "# id_class : all\n",
  2861. "# epoch : 100\n",
  2862. "# shift_tr : sharp\n",
  2863. "# crop : 0.08\n",
  2864. "# sharp : 3\n",
  2865. "# color_dist : 0.5\n",
  2866. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 3 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor3.0_one_class_0/last.model\""
  2867. ]
  2868. },
  2869. {
  2870. "cell_type": "code",
  2871. "execution_count": 85,
  2872. "id": "e8fde266",
  2873. "metadata": {},
  2874. "outputs": [
  2875. {
  2876. "name": "stdout",
  2877. "output_type": "stream",
  2878. "text": [
  2879. "Pre-compute global statistics...\n",
  2880. "axis size: 3527 3527 3527 3527\n",
  2881. "weight_sim:\t0.0051\t0.0046\t0.0048\t0.0045\n",
  2882. "weight_shi:\t-0.0137\t0.0407\t0.0450\t0.0411\n",
  2883. "Pre-compute features...\n",
  2884. "Compute OOD scores... (score: CSI)\n",
  2885. "One_class_real_mean: 0.4467181154356435\n",
  2886. "CNMC 2.0176 +- 0.0689 q0: 1.7903 q10: 1.9308 q20: 1.9620 q30: 1.9833 q40: 1.9998 q50: 2.0144 q60: 2.0329 q70: 2.0534 q80: 2.0752 q90: 2.1054 q100: 2.2461\n",
  2887. "one_class_1 2.0300 +- 0.0917 q0: 1.7417 q10: 1.9114 q20: 1.9580 q30: 1.9866 q40: 2.0089 q50: 2.0337 q60: 2.0591 q70: 2.0798 q80: 2.1052 q90: 2.1409 q100: 2.2672\n",
  2888. "[one_class_1 CSI 0.4467] [one_class_1 best 0.4467] \n",
  2889. "[one_class_mean CSI 0.4467] [one_class_mean best 0.4467] \n",
  2890. "0.4467\t0.4467\n"
  2891. ]
  2892. }
  2893. ],
  2894. "source": [
  2895. "###### EVALUATION\n",
  2896. "# dataset : CNMC\n",
  2897. "# res : 450px\n",
  2898. "# id_class : all\n",
  2899. "# epoch : 100\n",
  2900. "# shift_tr : sharp\n",
  2901. "# crop : 0.08\n",
  2902. "# sharp : 2\n",
  2903. "# color_dist : 0.5\n",
  2904. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --sharpness_factor 2 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type sharp --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/sharp/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_sharp_resize_factor0.08_color_dist0.5_sharpness_factor2.0_one_class_0/last.model\""
  2905. ]
  2906. },
  2907. {
  2908. "cell_type": "markdown",
  2909. "id": "bac55a6b",
  2910. "metadata": {},
  2911. "source": [
  2912. "# Random Perspective"
  2913. ]
  2914. },
  2915. {
  2916. "cell_type": "code",
  2917. "execution_count": 86,
  2918. "id": "acb8e0cf",
  2919. "metadata": {},
  2920. "outputs": [
  2921. {
  2922. "name": "stdout",
  2923. "output_type": "stream",
  2924. "text": [
  2925. "Pre-compute global statistics...\n",
  2926. "axis size: 3527 3527 3527 3527\n",
  2927. "weight_sim:\t0.0046\t0.0037\t0.0045\t0.0046\n",
  2928. "weight_shi:\t0.1028\t-0.1896\t-0.2910\t-0.3483\n",
  2929. "Pre-compute features...\n",
  2930. "Compute OOD scores... (score: CSI)\n",
  2931. "One_class_real_mean: 0.6587695844600411\n",
  2932. "CNMC 2.0386 +- 0.2243 q0: 1.1465 q10: 1.7463 q20: 1.8820 q30: 1.9641 q40: 2.0161 q50: 2.0690 q60: 2.1137 q70: 2.1678 q80: 2.2188 q90: 2.2888 q100: 2.5369\n",
  2933. "one_class_1 1.8805 +- 0.3066 q0: 0.7440 q10: 1.4384 q20: 1.6744 q30: 1.7753 q40: 1.8640 q50: 1.9389 q60: 1.9992 q70: 2.0643 q80: 2.1327 q90: 2.2159 q100: 2.4966\n",
  2934. "[one_class_1 CSI 0.6588] [one_class_1 best 0.6588] \n",
  2935. "[one_class_mean CSI 0.6588] [one_class_mean best 0.6588] \n",
  2936. "0.6588\t0.6588\n"
  2937. ]
  2938. }
  2939. ],
  2940. "source": [
  2941. "###### EVALUATION\n",
  2942. "# dataset : CNMC\n",
  2943. "# res : 450px\n",
  2944. "# id_class : all\n",
  2945. "# epoch : 100\n",
  2946. "# shift_tr : randpers\n",
  2947. "# crop : 0.08\n",
  2948. "# randper_dist: 0.95\n",
  2949. "# color_dist : 0.5\n",
  2950. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.95 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.95_one_class_0/last.model\""
  2951. ]
  2952. },
  2953. {
  2954. "cell_type": "code",
  2955. "execution_count": 87,
  2956. "id": "38406c45",
  2957. "metadata": {},
  2958. "outputs": [
  2959. {
  2960. "name": "stdout",
  2961. "output_type": "stream",
  2962. "text": [
  2963. "Pre-compute global statistics...\n",
  2964. "axis size: 3527 3527 3527 3527\n",
  2965. "weight_sim:\t0.0036\t0.0043\t0.0044\t0.0045\n",
  2966. "weight_shi:\t0.0940\t-0.3354\t-0.3010\t-0.4613\n",
  2967. "Pre-compute features...\n",
  2968. "Compute OOD scores... (score: CSI)\n",
  2969. "One_class_real_mean: 0.6866549691611218\n",
  2970. "CNMC 2.0701 +- 0.1458 q0: 1.3353 q10: 1.9029 q20: 1.9897 q30: 2.0319 q40: 2.0558 q50: 2.0795 q60: 2.1028 q70: 2.1319 q80: 2.1715 q90: 2.2439 q100: 2.4895\n",
  2971. "one_class_1 1.9579 +- 0.2070 q0: 0.9880 q10: 1.6843 q20: 1.8208 q30: 1.8972 q40: 1.9508 q50: 1.9923 q60: 2.0252 q70: 2.0622 q80: 2.0986 q90: 2.1886 q100: 2.4493\n",
  2972. "[one_class_1 CSI 0.6867] [one_class_1 best 0.6867] \n",
  2973. "[one_class_mean CSI 0.6867] [one_class_mean best 0.6867] \n",
  2974. "0.6867\t0.6867\n"
  2975. ]
  2976. }
  2977. ],
  2978. "source": [
  2979. "###### EVALUATION\n",
  2980. "# dataset : CNMC\n",
  2981. "# res : 450px\n",
  2982. "# id_class : all\n",
  2983. "# epoch : 100\n",
  2984. "# shift_tr : randpers\n",
  2985. "# crop : 0.08\n",
  2986. "# randper_dist: 0.9\n",
  2987. "# color_dist : 0.5\n",
  2988. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.9 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.9_one_class_0/last.model\""
  2989. ]
  2990. },
  2991. {
  2992. "cell_type": "code",
  2993. "execution_count": 88,
  2994. "id": "79e43776",
  2995. "metadata": {},
  2996. "outputs": [
  2997. {
  2998. "name": "stdout",
  2999. "output_type": "stream",
  3000. "text": [
  3001. "Pre-compute global statistics...\n",
  3002. "axis size: 3527 3527 3527 3527\n",
  3003. "weight_sim:\t0.0058\t0.0098\t0.0051\t0.0075\n",
  3004. "weight_shi:\t0.7573\t0.7158\t-0.4403\t3.1769\n",
  3005. "Pre-compute features...\n",
  3006. "Compute OOD scores... (score: CSI)\n",
  3007. "One_class_real_mean: 0.6343618023273478\n",
  3008. "CNMC 2.2964 +- 0.4564 q0: 0.6867 q10: 1.7046 q20: 1.9085 q30: 2.0619 q40: 2.1922 q50: 2.3256 q60: 2.4368 q70: 2.5468 q80: 2.6813 q90: 2.8610 q100: 3.7183\n",
  3009. "one_class_1 1.9670 +- 0.7117 q0: -1.6022 q10: 1.0639 q20: 1.4591 q30: 1.7160 q40: 1.8807 q50: 2.0547 q60: 2.2023 q70: 2.3776 q80: 2.5617 q90: 2.7902 q100: 3.2754\n",
  3010. "[one_class_1 CSI 0.6344] [one_class_1 best 0.6344] \n",
  3011. "[one_class_mean CSI 0.6344] [one_class_mean best 0.6344] \n",
  3012. "0.6344\t0.6344\n"
  3013. ]
  3014. }
  3015. ],
  3016. "source": [
  3017. "###### EVALUATION\n",
  3018. "# dataset : CNMC\n",
  3019. "# res : 450px\n",
  3020. "# id_class : all\n",
  3021. "# epoch : 100\n",
  3022. "# shift_tr : randpers\n",
  3023. "# crop : 0.08\n",
  3024. "# randper_dist: 0.85\n",
  3025. "# color_dist : 0.5\n",
  3026. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.85 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.85_one_class_0/last.model\""
  3027. ]
  3028. },
  3029. {
  3030. "cell_type": "code",
  3031. "execution_count": 89,
  3032. "id": "b5045a90",
  3033. "metadata": {},
  3034. "outputs": [
  3035. {
  3036. "name": "stdout",
  3037. "output_type": "stream",
  3038. "text": [
  3039. "Pre-compute global statistics...\n",
  3040. "axis size: 3527 3527 3527 3527\n",
  3041. "weight_sim:\t0.0044\t0.0059\t0.0035\t0.0046\n",
  3042. "weight_shi:\t0.1149\t-0.5921\t-0.2913\t-0.4212\n",
  3043. "Pre-compute features...\n",
  3044. "Compute OOD scores... (score: CSI)\n",
  3045. "One_class_real_mean: 0.6980790265244736\n",
  3046. "CNMC 2.0586 +- 0.0926 q0: 1.6907 q10: 1.9416 q20: 1.9891 q30: 2.0146 q40: 2.0367 q50: 2.0567 q60: 2.0821 q70: 2.1041 q80: 2.1311 q90: 2.1727 q100: 2.4236\n",
  3047. "one_class_1 1.9890 +- 0.1129 q0: 1.6747 q10: 1.8586 q20: 1.8947 q30: 1.9326 q40: 1.9579 q50: 1.9848 q60: 2.0103 q70: 2.0342 q80: 2.0767 q90: 2.1413 q100: 2.4328\n",
  3048. "[one_class_1 CSI 0.6981] [one_class_1 best 0.6981] \n",
  3049. "[one_class_mean CSI 0.6981] [one_class_mean best 0.6981] \n",
  3050. "0.6981\t0.6981\n"
  3051. ]
  3052. }
  3053. ],
  3054. "source": [
  3055. "###### EVALUATION\n",
  3056. "# dataset : CNMC\n",
  3057. "# res : 450px\n",
  3058. "# id_class : all\n",
  3059. "# epoch : 100\n",
  3060. "# shift_tr : randpers\n",
  3061. "# crop : 0.08\n",
  3062. "# randper_dist: 0.8\n",
  3063. "# color_dist : 0.5\n",
  3064. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.8 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.8_one_class_0/last.model\""
  3065. ]
  3066. },
  3067. {
  3068. "cell_type": "code",
  3069. "execution_count": 90,
  3070. "id": "5d4659ac",
  3071. "metadata": {},
  3072. "outputs": [
  3073. {
  3074. "name": "stdout",
  3075. "output_type": "stream",
  3076. "text": [
  3077. "Pre-compute global statistics...\n",
  3078. "axis size: 3527 3527 3527 3527\n",
  3079. "weight_sim:\t0.0020\t0.0024\t0.0019\t0.0028\n",
  3080. "weight_shi:\t0.0839\t-0.1992\t-0.1714\t-0.2720\n",
  3081. "Pre-compute features...\n",
  3082. "Compute OOD scores... (score: CSI)\n",
  3083. "One_class_real_mean: 0.7161709152411915\n",
  3084. "CNMC 2.0766 +- 0.1562 q0: 1.4380 q10: 1.8760 q20: 1.9670 q30: 2.0275 q40: 2.0724 q50: 2.1054 q60: 2.1342 q70: 2.1645 q80: 2.2012 q90: 2.2452 q100: 2.4108\n",
  3085. "one_class_1 1.9367 +- 0.2053 q0: 1.1357 q10: 1.6437 q20: 1.7718 q30: 1.8741 q40: 1.9299 q50: 1.9756 q60: 2.0202 q70: 2.0576 q80: 2.1027 q90: 2.1710 q100: 2.3247\n",
  3086. "[one_class_1 CSI 0.7162] [one_class_1 best 0.7162] \n",
  3087. "[one_class_mean CSI 0.7162] [one_class_mean best 0.7162] \n",
  3088. "0.7162\t0.7162\n"
  3089. ]
  3090. }
  3091. ],
  3092. "source": [
  3093. "###### EVALUATION\n",
  3094. "# dataset : CNMC\n",
  3095. "# res : 450px\n",
  3096. "# id_class : all\n",
  3097. "# epoch : 100\n",
  3098. "# shift_tr : randpers\n",
  3099. "# crop : 0.08\n",
  3100. "# randper_dist: 0.75\n",
  3101. "# color_dist : 0.5\n",
  3102. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.75 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.75_one_class_0/last.model\""
  3103. ]
  3104. },
  3105. {
  3106. "cell_type": "code",
  3107. "execution_count": 91,
  3108. "id": "43c01d76",
  3109. "metadata": {},
  3110. "outputs": [
  3111. {
  3112. "name": "stdout",
  3113. "output_type": "stream",
  3114. "text": [
  3115. "Pre-compute global statistics...\n",
  3116. "axis size: 3527 3527 3527 3527\n",
  3117. "weight_sim:\t0.0049\t0.0060\t0.0045\t0.0072\n",
  3118. "weight_shi:\t-1.4937\t0.4193\t-0.5923\t0.9519\n",
  3119. "Pre-compute features...\n",
  3120. "Compute OOD scores... (score: CSI)\n",
  3121. "One_class_real_mean: 0.272612645459241\n",
  3122. "CNMC 1.3428 +- 1.3765 q0: -2.6822 q10: -0.1317 q20: 0.2430 q30: 0.5766 q40: 0.8777 q50: 1.1651 q60: 1.4791 q70: 1.7858 q80: 2.3163 q90: 3.1566 q100: 7.5281\n",
  3123. "one_class_1 2.6219 +- 1.7311 q0: -1.5026 q10: 0.6189 q20: 1.1608 q30: 1.6420 q40: 2.0065 q50: 2.4546 q60: 2.8590 q70: 3.3026 q80: 3.9411 q90: 5.0125 q100: 8.8420\n",
  3124. "[one_class_1 CSI 0.2726] [one_class_1 best 0.2726] \n",
  3125. "[one_class_mean CSI 0.2726] [one_class_mean best 0.2726] \n",
  3126. "0.2726\t0.2726\n"
  3127. ]
  3128. }
  3129. ],
  3130. "source": [
  3131. "###### EVALUATION\n",
  3132. "# dataset : CNMC\n",
  3133. "# res : 450px\n",
  3134. "# id_class : all\n",
  3135. "# epoch : 100\n",
  3136. "# shift_tr : randpers\n",
  3137. "# crop : 0.08\n",
  3138. "# randper_dist: 0.6\n",
  3139. "# color_dist : 0.5\n",
  3140. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.6 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.6_one_class_0/last.model\""
  3141. ]
  3142. },
  3143. {
  3144. "cell_type": "code",
  3145. "execution_count": 92,
  3146. "id": "b3c2bb68",
  3147. "metadata": {},
  3148. "outputs": [
  3149. {
  3150. "name": "stdout",
  3151. "output_type": "stream",
  3152. "text": [
  3153. "Pre-compute global statistics...\n",
  3154. "axis size: 3527 3527 3527 3527\n",
  3155. "weight_sim:\t0.0040\t0.0080\t0.0069\t0.0094\n",
  3156. "weight_shi:\t0.2243\t2.4831\t-1.1810\t8.3228\n",
  3157. "Pre-compute features...\n",
  3158. "Compute OOD scores... (score: CSI)\n",
  3159. "One_class_real_mean: 0.28934109115952156\n",
  3160. "CNMC 1.5171 +- 0.6417 q0: -1.0892 q10: 0.6963 q20: 1.0163 q30: 1.2147 q40: 1.3800 q50: 1.5239 q60: 1.6610 q70: 1.8232 q80: 2.0413 q90: 2.3147 q100: 3.5528\n",
  3161. "one_class_1 2.0698 +- 0.8043 q0: -0.3686 q10: 0.9775 q20: 1.4094 q30: 1.6626 q40: 1.9406 q50: 2.1430 q60: 2.3291 q70: 2.4725 q80: 2.7776 q90: 3.0563 q100: 4.2509\n",
  3162. "[one_class_1 CSI 0.2893] [one_class_1 best 0.2893] \n",
  3163. "[one_class_mean CSI 0.2893] [one_class_mean best 0.2893] \n",
  3164. "0.2893\t0.2893\n"
  3165. ]
  3166. }
  3167. ],
  3168. "source": [
  3169. "###### EVALUATION\n",
  3170. "# dataset : CNMC\n",
  3171. "# res : 450px\n",
  3172. "# id_class : all\n",
  3173. "# epoch : 100\n",
  3174. "# shift_tr : randpers\n",
  3175. "# crop : 0.08\n",
  3176. "# randper_dist: 0.3\n",
  3177. "# color_dist : 0.5\n",
  3178. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --distortion_scale 0.3 --color_distort 0.5 --resize_factor 0.08 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type randpers --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/randpers/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_randpers_resize_factor0.08_color_dist0.5_distortion_scale0.3_one_class_0/last.model\""
  3179. ]
  3180. },
  3181. {
  3182. "cell_type": "markdown",
  3183. "id": "5cfed222",
  3184. "metadata": {},
  3185. "source": [
  3186. "# Color Distortion = 0.8"
  3187. ]
  3188. },
  3189. {
  3190. "cell_type": "markdown",
  3191. "id": "009f41d0",
  3192. "metadata": {},
  3193. "source": [
  3194. "## Examine crop"
  3195. ]
  3196. },
  3197. {
  3198. "cell_type": "code",
  3199. "execution_count": 196,
  3200. "id": "0c216c1d",
  3201. "metadata": {},
  3202. "outputs": [
  3203. {
  3204. "name": "stdout",
  3205. "output_type": "stream",
  3206. "text": [
  3207. "Pre-compute global statistics...\n",
  3208. "axis size: 3527 3527 3527 3527\n",
  3209. "weight_sim:\t0.0109\t0.0127\t0.0131\t0.0112\n",
  3210. "weight_shi:\t-0.3601\t0.8696\t0.8266\t1.2633\n",
  3211. "Pre-compute features...\n",
  3212. "Compute OOD scores... (score: CSI)\n",
  3213. "One_class_real_mean: 0.3773116499053059\n",
  3214. "CNMC 1.9315 +- 0.0895 q0: 1.6857 q10: 1.8069 q20: 1.8435 q30: 1.8794 q40: 1.9068 q50: 1.9359 q60: 1.9582 q70: 1.9849 q80: 2.0138 q90: 2.0465 q100: 2.2963\n",
  3215. "one_class_1 1.9880 +- 0.1336 q0: 1.7198 q10: 1.8075 q20: 1.8598 q30: 1.9006 q40: 1.9419 q50: 1.9827 q60: 2.0260 q70: 2.0729 q80: 2.1099 q90: 2.1645 q100: 2.4634\n",
  3216. "[one_class_1 CSI 0.3773] [one_class_1 best 0.3773] \n",
  3217. "[one_class_mean CSI 0.3773] [one_class_mean best 0.3773] \n",
  3218. "0.3773\t0.3773\n"
  3219. ]
  3220. }
  3221. ],
  3222. "source": [
  3223. "# EVALUATION\n",
  3224. "# dataset : CNMC\n",
  3225. "# res : 450px\n",
  3226. "# id_class : all\n",
  3227. "# epoch : 100\n",
  3228. "# shift_tr : blur\n",
  3229. "# crop : 0.5\n",
  3230. "# blur_sigma : 2\n",
  3231. "# color_dist : 0.8\n",
  3232. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.5 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.5_color_dist0.8_blur_sigma2.0_one_class_0/last.model\""
  3233. ]
  3234. },
  3235. {
  3236. "cell_type": "code",
  3237. "execution_count": 197,
  3238. "id": "6320eef5",
  3239. "metadata": {},
  3240. "outputs": [
  3241. {
  3242. "name": "stdout",
  3243. "output_type": "stream",
  3244. "text": [
  3245. "Pre-compute global statistics...\n",
  3246. "axis size: 3527 3527 3527 3527\n",
  3247. "weight_sim:\t0.0076\t0.0076\t0.0074\t0.0074\n",
  3248. "weight_shi:\t0.9058\t0.5362\t0.6368\t-14.1887\n",
  3249. "Pre-compute features...\n",
  3250. "Compute OOD scores... (score: CSI)\n",
  3251. "One_class_real_mean: 0.5917127477491163\n",
  3252. "CNMC 2.8214 +- 1.0315 q0: -1.2486 q10: 1.3987 q20: 1.9532 q30: 2.3459 q40: 2.7058 q50: 2.9513 q60: 3.1875 q70: 3.4447 q80: 3.7111 q90: 4.0288 q100: 5.9040\n",
  3253. "one_class_1 2.3812 +- 1.3314 q0: -2.1268 q10: 0.5397 q20: 1.2181 q30: 1.7456 q40: 2.2246 q50: 2.5793 q60: 2.9176 q70: 3.1949 q80: 3.5267 q90: 3.9124 q100: 4.9106\n",
  3254. "[one_class_1 CSI 0.5917] [one_class_1 best 0.5917] \n",
  3255. "[one_class_mean CSI 0.5917] [one_class_mean best 0.5917] \n",
  3256. "0.5917\t0.5917\n"
  3257. ]
  3258. }
  3259. ],
  3260. "source": [
  3261. "# EVALUATION\n",
  3262. "# dataset : CNMC\n",
  3263. "# res : 450px\n",
  3264. "# id_class : all\n",
  3265. "# epoch : 100\n",
  3266. "# shift_tr : blur\n",
  3267. "# crop : 0.3\n",
  3268. "# blur_sigma : 2\n",
  3269. "# color_dist : 0.8\n",
  3270. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.3 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.3_color_dist0.8_blur_sigma2.0_one_class_0/last.model\""
  3271. ]
  3272. },
  3273. {
  3274. "cell_type": "code",
  3275. "execution_count": 198,
  3276. "id": "451c90e5",
  3277. "metadata": {},
  3278. "outputs": [
  3279. {
  3280. "name": "stdout",
  3281. "output_type": "stream",
  3282. "text": [
  3283. "Pre-compute global statistics...\n",
  3284. "axis size: 3527 3527 3527 3527\n",
  3285. "weight_sim:\t0.0110\t0.0071\t0.0102\t0.0101\n",
  3286. "weight_shi:\t-0.2335\t0.3455\t0.5920\t0.5756\n",
  3287. "Pre-compute features...\n",
  3288. "Compute OOD scores... (score: CSI)\n",
  3289. "One_class_real_mean: 0.4148462107171432\n",
  3290. "CNMC 1.8587 +- 0.2019 q0: 1.3001 q10: 1.5984 q20: 1.6837 q30: 1.7477 q40: 1.8032 q50: 1.8568 q60: 1.9066 q70: 1.9618 q80: 2.0281 q90: 2.1033 q100: 2.4803\n",
  3291. "one_class_1 1.9549 +- 0.3011 q0: 1.3374 q10: 1.5829 q20: 1.6833 q30: 1.7648 q40: 1.8483 q50: 1.9432 q60: 2.0105 q70: 2.0860 q80: 2.2017 q90: 2.3676 q100: 3.0008\n",
  3292. "[one_class_1 CSI 0.4148] [one_class_1 best 0.4148] \n",
  3293. "[one_class_mean CSI 0.4148] [one_class_mean best 0.4148] \n",
  3294. "0.4148\t0.4148\n"
  3295. ]
  3296. }
  3297. ],
  3298. "source": [
  3299. "# EVALUATION\n",
  3300. "# dataset : CNMC\n",
  3301. "# res : 450px\n",
  3302. "# id_class : all\n",
  3303. "# epoch : 100\n",
  3304. "# shift_tr : blur\n",
  3305. "# crop : 0.02\n",
  3306. "# blur_sigma : 2\n",
  3307. "# color_dist : 0.8\n",
  3308. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.02 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.02_color_dist0.8_blur_sigma2.0_one_class_0/last.model\""
  3309. ]
  3310. },
  3311. {
  3312. "cell_type": "code",
  3313. "execution_count": 199,
  3314. "id": "54fef60e",
  3315. "metadata": {},
  3316. "outputs": [
  3317. {
  3318. "name": "stdout",
  3319. "output_type": "stream",
  3320. "text": [
  3321. "Pre-compute global statistics...\n",
  3322. "axis size: 3527 3527 3527 3527\n",
  3323. "weight_sim:\t0.0055\t0.0047\t0.0063\t0.0070\n",
  3324. "weight_shi:\t-1.5156\t2.2142\t13.3925\t216.9532\n",
  3325. "Pre-compute features...\n",
  3326. "Compute OOD scores... (score: CSI)\n",
  3327. "One_class_real_mean: 0.3877887663435927\n",
  3328. "CNMC -9.2248 +- 14.9978 q0: -31.8010 q10: -22.8496 q20: -20.5631 q30: -18.8369 q40: -16.1600 q50: -13.7478 q60: -10.1906 q70: -5.6572 q80: 0.0581 q90: 9.1230 q100: 77.4578\n",
  3329. "one_class_1 0.7817 +- 24.0001 q0: -33.6751 q10: -22.8505 q20: -20.2689 q30: -16.3248 q40: -11.6706 q50: -5.3667 q60: 0.3825 q70: 6.7728 q80: 17.5805 q90: 39.7293 q100: 83.7649\n",
  3330. "[one_class_1 CSI 0.3878] [one_class_1 best 0.3878] \n",
  3331. "[one_class_mean CSI 0.3878] [one_class_mean best 0.3878] \n",
  3332. "0.3878\t0.3878\n"
  3333. ]
  3334. }
  3335. ],
  3336. "source": [
  3337. "# EVALUATION\n",
  3338. "# dataset : CNMC\n",
  3339. "# res : 450px\n",
  3340. "# id_class : all\n",
  3341. "# epoch : 100\n",
  3342. "# shift_tr : blur\n",
  3343. "# crop : 0.008\n",
  3344. "# blur_sigma : 2\n",
  3345. "# color_dist : 0.8\n",
  3346. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.008 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.008_color_dist0.8_blur_sigma2.0_one_class_0/last.model\""
  3347. ]
  3348. },
  3349. {
  3350. "cell_type": "markdown",
  3351. "id": "2dccb685",
  3352. "metadata": {},
  3353. "source": [
  3354. "## Examine blur_sigma"
  3355. ]
  3356. },
  3357. {
  3358. "cell_type": "code",
  3359. "execution_count": 200,
  3360. "id": "0c13892c",
  3361. "metadata": {},
  3362. "outputs": [
  3363. {
  3364. "name": "stdout",
  3365. "output_type": "stream",
  3366. "text": [
  3367. "Pre-compute global statistics...\n",
  3368. "axis size: 3527 3527 3527 3527\n",
  3369. "weight_sim:\t0.0059\t0.0053\t0.0052\t0.0054\n",
  3370. "weight_shi:\t-0.0908\t0.2339\t0.2553\t0.2459\n",
  3371. "Pre-compute features...\n",
  3372. "Compute OOD scores... (score: CSI)\n",
  3373. "One_class_real_mean: 0.46215401209248624\n",
  3374. "CNMC 1.9646 +- 0.0814 q0: 1.7239 q10: 1.8537 q20: 1.8937 q30: 1.9247 q40: 1.9510 q50: 1.9695 q60: 1.9918 q70: 2.0122 q80: 2.0334 q90: 2.0642 q100: 2.1895\n",
  3375. "one_class_1 1.9790 +- 0.1048 q0: 1.6906 q10: 1.8438 q20: 1.8841 q30: 1.9178 q40: 1.9505 q50: 1.9783 q60: 2.0103 q70: 2.0393 q80: 2.0700 q90: 2.1155 q100: 2.2617\n",
  3376. "[one_class_1 CSI 0.4622] [one_class_1 best 0.4622] \n",
  3377. "[one_class_mean CSI 0.4622] [one_class_mean best 0.4622] \n",
  3378. "0.4622\t0.4622\n"
  3379. ]
  3380. }
  3381. ],
  3382. "source": [
  3383. "# EVALUATION\n",
  3384. "# dataset : CNMC\n",
  3385. "# res : 450px\n",
  3386. "# id_class : all\n",
  3387. "# epoch : 100\n",
  3388. "# shift_tr : blur\n",
  3389. "# crop : 0.08\n",
  3390. "# blur_sigma : 40\n",
  3391. "# color_dist : 0.8 \n",
  3392. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 40 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma40.0_one_class_0/last.model\""
  3393. ]
  3394. },
  3395. {
  3396. "cell_type": "code",
  3397. "execution_count": 201,
  3398. "id": "7b24db11",
  3399. "metadata": {},
  3400. "outputs": [
  3401. {
  3402. "name": "stdout",
  3403. "output_type": "stream",
  3404. "text": [
  3405. "Pre-compute global statistics...\n",
  3406. "axis size: 3527 3527 3527 3527\n",
  3407. "weight_sim:\t0.0172\t0.0135\t0.0226\t0.0192\n",
  3408. "weight_shi:\t-0.0741\t0.1495\t0.1978\t0.1718\n",
  3409. "Pre-compute features...\n",
  3410. "Compute OOD scores... (score: CSI)\n",
  3411. "One_class_real_mean: 0.4793875773503884\n",
  3412. "CNMC 1.9531 +- 0.1138 q0: 1.6474 q10: 1.8092 q20: 1.8542 q30: 1.8878 q40: 1.9235 q50: 1.9532 q60: 1.9802 q70: 2.0102 q80: 2.0447 q90: 2.0976 q100: 2.3859\n",
  3413. "one_class_1 1.9692 +- 0.1523 q0: 1.6030 q10: 1.7796 q20: 1.8318 q30: 1.8785 q40: 1.9130 q50: 1.9548 q60: 1.9983 q70: 2.0455 q80: 2.0996 q90: 2.1848 q100: 2.3561\n",
  3414. "[one_class_1 CSI 0.4794] [one_class_1 best 0.4794] \n",
  3415. "[one_class_mean CSI 0.4794] [one_class_mean best 0.4794] \n",
  3416. "0.4794\t0.4794\n"
  3417. ]
  3418. }
  3419. ],
  3420. "source": [
  3421. "# EVALUATION\n",
  3422. "# dataset : CNMC\n",
  3423. "# res : 450px\n",
  3424. "# id_class : all\n",
  3425. "# epoch : 100\n",
  3426. "# shift_tr : blur\n",
  3427. "# crop : 0.08\n",
  3428. "# blur_sigma : 20\n",
  3429. "# color_dist : 0.8 \n",
  3430. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 20 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma20.0_one_class_0/last.model\""
  3431. ]
  3432. },
  3433. {
  3434. "cell_type": "code",
  3435. "execution_count": 202,
  3436. "id": "352c0a41",
  3437. "metadata": {},
  3438. "outputs": [
  3439. {
  3440. "name": "stdout",
  3441. "output_type": "stream",
  3442. "text": [
  3443. "Pre-compute global statistics...\n",
  3444. "axis size: 3527 3527 3527 3527\n",
  3445. "weight_sim:\t0.0025\t0.0040\t0.0022\t0.0025\n",
  3446. "weight_shi:\t1.2410\t0.6755\t-1.1582\t-3.5877\n",
  3447. "Pre-compute features...\n",
  3448. "Compute OOD scores... (score: CSI)\n",
  3449. "One_class_real_mean: 0.6594658645520007\n",
  3450. "CNMC 2.1628 +- 0.3004 q0: 0.7838 q10: 1.7714 q20: 1.9641 q30: 2.0468 q40: 2.1213 q50: 2.1830 q60: 2.2509 q70: 2.3213 q80: 2.4001 q90: 2.5014 q100: 3.3713\n",
  3451. "one_class_1 1.9514 +- 0.4284 q0: -0.1104 q10: 1.4546 q20: 1.6765 q30: 1.7941 q40: 1.8892 q50: 2.0071 q60: 2.0917 q70: 2.1785 q80: 2.2711 q90: 2.4356 q100: 2.9090\n",
  3452. "[one_class_1 CSI 0.6595] [one_class_1 best 0.6595] \n",
  3453. "[one_class_mean CSI 0.6595] [one_class_mean best 0.6595] \n",
  3454. "0.6595\t0.6595\n"
  3455. ]
  3456. }
  3457. ],
  3458. "source": [
  3459. "# EVALUATION\n",
  3460. "# dataset : CNMC\n",
  3461. "# res : 450px\n",
  3462. "# id_class : all\n",
  3463. "# epoch : 100\n",
  3464. "# shift_tr : blur\n",
  3465. "# crop : 0.08\n",
  3466. "# blur_sigma : 6\n",
  3467. "# color_dist : 0.8 \n",
  3468. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 6 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma6.0_one_class_0/last.model\""
  3469. ]
  3470. },
  3471. {
  3472. "cell_type": "code",
  3473. "execution_count": 203,
  3474. "id": "d22c485a",
  3475. "metadata": {},
  3476. "outputs": [
  3477. {
  3478. "name": "stdout",
  3479. "output_type": "stream",
  3480. "text": [
  3481. "Pre-compute global statistics...\n",
  3482. "axis size: 3527 3527 3527 3527\n",
  3483. "weight_sim:\t0.0050\t0.0078\t0.0052\t0.0062\n",
  3484. "weight_shi:\t0.4106\t0.4163\t-2.7425\t-3.5688\n",
  3485. "Pre-compute features...\n",
  3486. "Compute OOD scores... (score: CSI)\n",
  3487. "One_class_real_mean: 0.6272255643666637\n",
  3488. "CNMC 2.1200 +- 0.6206 q0: -0.0284 q10: 1.2949 q20: 1.5957 q30: 1.8155 q40: 2.0312 q50: 2.1755 q60: 2.3270 q70: 2.4960 q80: 2.6611 q90: 2.8534 q100: 4.1276\n",
  3489. "one_class_1 1.8269 +- 0.6667 q0: -0.9521 q10: 0.9740 q20: 1.3397 q30: 1.5917 q40: 1.7532 q50: 1.8870 q60: 2.0460 q70: 2.2002 q80: 2.3477 q90: 2.5711 q100: 3.5803\n",
  3490. "[one_class_1 CSI 0.6272] [one_class_1 best 0.6272] \n",
  3491. "[one_class_mean CSI 0.6272] [one_class_mean best 0.6272] \n",
  3492. "0.6272\t0.6272\n"
  3493. ]
  3494. }
  3495. ],
  3496. "source": [
  3497. "# EVALUATION\n",
  3498. "# dataset : CNMC\n",
  3499. "# res : 450px\n",
  3500. "# id_class : all\n",
  3501. "# epoch : 100\n",
  3502. "# shift_tr : blur\n",
  3503. "# crop : 0.08\n",
  3504. "# blur_sigma : 4\n",
  3505. "# color_dist : 0.8 \n",
  3506. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 4 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma4.0_one_class_0/last.model\""
  3507. ]
  3508. },
  3509. {
  3510. "cell_type": "code",
  3511. "execution_count": 204,
  3512. "id": "00a8d2ac",
  3513. "metadata": {},
  3514. "outputs": [
  3515. {
  3516. "name": "stdout",
  3517. "output_type": "stream",
  3518. "text": [
  3519. "Pre-compute global statistics...\n",
  3520. "axis size: 3527 3527 3527 3527\n",
  3521. "weight_sim:\t0.0047\t0.0072\t0.0056\t0.0047\n",
  3522. "weight_shi:\t0.3757\t1.6655\t5.1831\t-1.0361\n",
  3523. "Pre-compute features...\n",
  3524. "Compute OOD scores... (score: CSI)\n",
  3525. "One_class_real_mean: 0.49718323053707253\n",
  3526. "CNMC 1.9914 +- 0.1852 q0: 1.3248 q10: 1.7612 q20: 1.8463 q30: 1.9021 q40: 1.9490 q50: 1.9936 q60: 2.0285 q70: 2.0851 q80: 2.1355 q90: 2.2233 q100: 2.6945\n",
  3527. "one_class_1 1.9965 +- 0.2132 q0: 1.1855 q10: 1.7487 q20: 1.8267 q30: 1.8843 q40: 1.9394 q50: 1.9881 q60: 2.0429 q70: 2.1000 q80: 2.1632 q90: 2.2481 q100: 2.8524\n",
  3528. "[one_class_1 CSI 0.4972] [one_class_1 best 0.4972] \n",
  3529. "[one_class_mean CSI 0.4972] [one_class_mean best 0.4972] \n",
  3530. "0.4972\t0.4972\n"
  3531. ]
  3532. }
  3533. ],
  3534. "source": [
  3535. "# EVALUATION\n",
  3536. "# dataset : CNMC\n",
  3537. "# res : 450px\n",
  3538. "# id_class : all\n",
  3539. "# epoch : 100\n",
  3540. "# shift_tr : blur\n",
  3541. "# crop : 0.08\n",
  3542. "# blur_sigma : 3\n",
  3543. "# color_dist : 0.8 \n",
  3544. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 3 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma3.0_one_class_0/last.model\""
  3545. ]
  3546. },
  3547. {
  3548. "cell_type": "code",
  3549. "execution_count": 205,
  3550. "id": "cdab5a91",
  3551. "metadata": {},
  3552. "outputs": [
  3553. {
  3554. "name": "stdout",
  3555. "output_type": "stream",
  3556. "text": [
  3557. "Pre-compute global statistics...\n",
  3558. "axis size: 3527 3527 3527 3527\n",
  3559. "weight_sim:\t0.0019\t0.0026\t0.0019\t0.0022\n",
  3560. "weight_shi:\t0.2520\t-1.0379\t-0.8245\t-0.8299\n",
  3561. "Pre-compute features...\n",
  3562. "Compute OOD scores... (score: CSI)\n",
  3563. "One_class_real_mean: 0.7393317230273752\n",
  3564. "CNMC 2.1093 +- 0.1696 q0: 1.2392 q10: 1.9016 q20: 2.0068 q30: 2.0585 q40: 2.1066 q50: 2.1333 q60: 2.1667 q70: 2.1958 q80: 2.2352 q90: 2.2885 q100: 2.5315\n",
  3565. "one_class_1 1.9282 +- 0.2660 q0: 0.4865 q10: 1.6153 q20: 1.7843 q30: 1.8731 q40: 1.9295 q50: 1.9714 q60: 2.0108 q70: 2.0668 q80: 2.1224 q90: 2.2106 q100: 2.5011\n",
  3566. "[one_class_1 CSI 0.7393] [one_class_1 best 0.7393] \n",
  3567. "[one_class_mean CSI 0.7393] [one_class_mean best 0.7393] \n",
  3568. "0.7393\t0.7393\n"
  3569. ]
  3570. }
  3571. ],
  3572. "source": [
  3573. "# EVALUATION\n",
  3574. "# dataset : CNMC\n",
  3575. "# res : 450px\n",
  3576. "# id_class : all\n",
  3577. "# epoch : 100\n",
  3578. "# shift_tr : blur\n",
  3579. "# crop : 0.08\n",
  3580. "# blur_sigma : 2\n",
  3581. "# color_dist : 0.8 \n",
  3582. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma2.0_one_class_0/last.model\""
  3583. ]
  3584. },
  3585. {
  3586. "cell_type": "code",
  3587. "execution_count": 206,
  3588. "id": "76bdab2e",
  3589. "metadata": {},
  3590. "outputs": [
  3591. {
  3592. "name": "stdout",
  3593. "output_type": "stream",
  3594. "text": [
  3595. "Pre-compute global statistics...\n",
  3596. "axis size: 3527 3527 3527 3527\n",
  3597. "weight_sim:\t0.0044\t0.0059\t0.0046\t0.0046\n",
  3598. "weight_shi:\t0.2676\t-0.5492\t-0.7697\t-0.6319\n",
  3599. "Pre-compute features...\n",
  3600. "Compute OOD scores... (score: CSI)\n",
  3601. "One_class_real_mean: 0.6709070124267007\n",
  3602. "CNMC 2.0490 +- 0.0659 q0: 1.6783 q10: 1.9728 q20: 1.9995 q30: 2.0151 q40: 2.0347 q50: 2.0501 q60: 2.0654 q70: 2.0818 q80: 2.1012 q90: 2.1296 q100: 2.2543\n",
  3603. "one_class_1 1.9948 +- 0.1054 q0: 1.5066 q10: 1.8893 q20: 1.9323 q30: 1.9563 q40: 1.9732 q50: 1.9993 q60: 2.0219 q70: 2.0484 q80: 2.0819 q90: 2.1226 q100: 2.2211\n",
  3604. "[one_class_1 CSI 0.6709] [one_class_1 best 0.6709] \n",
  3605. "[one_class_mean CSI 0.6709] [one_class_mean best 0.6709] \n",
  3606. "0.6709\t0.6709\n"
  3607. ]
  3608. }
  3609. ],
  3610. "source": [
  3611. "# EVALUATION\n",
  3612. "# dataset : CNMC\n",
  3613. "# res : 450px\n",
  3614. "# id_class : all\n",
  3615. "# epoch : 100\n",
  3616. "# shift_tr : blur\n",
  3617. "# crop : 0.08\n",
  3618. "# blur_sigma : 1.5\n",
  3619. "# color_dist : 0.8 \n",
  3620. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 1.5 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma1.5_one_class_0/last.model\""
  3621. ]
  3622. },
  3623. {
  3624. "cell_type": "code",
  3625. "execution_count": 207,
  3626. "id": "0c1efb9f",
  3627. "metadata": {},
  3628. "outputs": [
  3629. {
  3630. "name": "stdout",
  3631. "output_type": "stream",
  3632. "text": [
  3633. "Pre-compute global statistics...\n",
  3634. "axis size: 3527 3527 3527 3527\n",
  3635. "weight_sim:\t0.0058\t0.0159\t0.0080\t0.0086\n",
  3636. "weight_shi:\t0.5438\t-2.8363\t-21.1928\t-1.9421\n",
  3637. "Pre-compute features...\n",
  3638. "Compute OOD scores... (score: CSI)\n",
  3639. "One_class_real_mean: 0.6621902186572681\n",
  3640. "CNMC 2.2408 +- 0.8449 q0: -0.8843 q10: 1.1519 q20: 1.6038 q30: 1.9076 q40: 2.1231 q50: 2.3416 q60: 2.5656 q70: 2.7323 q80: 2.9607 q90: 3.2092 q100: 3.9264\n",
  3641. "one_class_1 1.6402 +- 1.1251 q0: -2.9414 q10: 0.0207 q20: 0.8939 q30: 1.3058 q40: 1.6627 q50: 1.9102 q60: 2.1044 q70: 2.2975 q80: 2.5539 q90: 2.8038 q100: 3.7386\n",
  3642. "[one_class_1 CSI 0.6622] [one_class_1 best 0.6622] \n",
  3643. "[one_class_mean CSI 0.6622] [one_class_mean best 0.6622] \n",
  3644. "0.6622\t0.6622\n"
  3645. ]
  3646. }
  3647. ],
  3648. "source": [
  3649. "# EVALUATION\n",
  3650. "# dataset : CNMC\n",
  3651. "# res : 450px\n",
  3652. "# id_class : all\n",
  3653. "# epoch : 100\n",
  3654. "# shift_tr : blur\n",
  3655. "# crop : 0.08\n",
  3656. "# blur_sigma : 1\n",
  3657. "# color_dist : 0.8 \n",
  3658. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.8 --resize_factor 0.08 --blur_sigma 1 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.8/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.8_blur_sigma1.0_one_class_0/last.model\""
  3659. ]
  3660. },
  3661. {
  3662. "cell_type": "markdown",
  3663. "id": "f676267b",
  3664. "metadata": {},
  3665. "source": [
  3666. "# Color Distortion = 1"
  3667. ]
  3668. },
  3669. {
  3670. "cell_type": "markdown",
  3671. "id": "744297b9",
  3672. "metadata": {},
  3673. "source": [
  3674. "## Examine crop"
  3675. ]
  3676. },
  3677. {
  3678. "cell_type": "code",
  3679. "execution_count": 208,
  3680. "id": "21a87be2",
  3681. "metadata": {
  3682. "scrolled": true
  3683. },
  3684. "outputs": [
  3685. {
  3686. "name": "stdout",
  3687. "output_type": "stream",
  3688. "text": [
  3689. "Pre-compute global statistics...\n",
  3690. "axis size: 3527 3527 3527 3527\n",
  3691. "weight_sim:\t0.0061\t0.0065\t0.0065\t0.0056\n",
  3692. "weight_shi:\t1.6932\t-31.1268\t15.0080\t-10.2414\n",
  3693. "Pre-compute features...\n",
  3694. "Compute OOD scores... (score: CSI)\n",
  3695. "One_class_real_mean: 0.7102132895816242\n",
  3696. "CNMC 2.2522 +- 0.5226 q0: -0.3755 q10: 1.6169 q20: 1.9232 q30: 2.1160 q40: 2.2220 q50: 2.3245 q60: 2.4225 q70: 2.5144 q80: 2.6312 q90: 2.8060 q100: 3.9139\n",
  3697. "one_class_1 1.8127 +- 0.7110 q0: -1.7832 q10: 0.9329 q20: 1.3309 q30: 1.6150 q40: 1.7793 q50: 1.9225 q60: 2.0429 q70: 2.1887 q80: 2.3378 q90: 2.5668 q100: 3.4155\n",
  3698. "[one_class_1 CSI 0.7102] [one_class_1 best 0.7102] \n",
  3699. "[one_class_mean CSI 0.7102] [one_class_mean best 0.7102] \n",
  3700. "0.7102\t0.7102\n"
  3701. ]
  3702. }
  3703. ],
  3704. "source": [
  3705. "# EVALUATION\n",
  3706. "# dataset : CNMC\n",
  3707. "# res : 450px\n",
  3708. "# id_class : all\n",
  3709. "# epoch : 100\n",
  3710. "# shift_tr : blur\n",
  3711. "# crop : 0.5\n",
  3712. "# blur_sigma : 2\n",
  3713. "# color_dist : 1\n",
  3714. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.5 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.5_color_dist1.0_one_class_0/last.model\""
  3715. ]
  3716. },
  3717. {
  3718. "cell_type": "code",
  3719. "execution_count": 209,
  3720. "id": "8dd1d6d5",
  3721. "metadata": {
  3722. "scrolled": true
  3723. },
  3724. "outputs": [
  3725. {
  3726. "name": "stdout",
  3727. "output_type": "stream",
  3728. "text": [
  3729. "Pre-compute global statistics...\n",
  3730. "axis size: 3527 3527 3527 3527\n",
  3731. "weight_sim:\t0.0092\t0.0099\t0.0099\t0.0096\n",
  3732. "weight_shi:\t0.5734\t-1.4904\t-1.4266\t-2.6760\n",
  3733. "Pre-compute features...\n",
  3734. "Compute OOD scores... (score: CSI)\n",
  3735. "One_class_real_mean: 0.5938636202513699\n",
  3736. "CNMC 2.0102 +- 0.1072 q0: 1.4470 q10: 1.8844 q20: 1.9436 q30: 1.9761 q40: 2.0084 q50: 2.0343 q60: 2.0578 q70: 2.0757 q80: 2.0944 q90: 2.1139 q100: 2.2014\n",
  3737. "one_class_1 1.9687 +- 0.1365 q0: 1.2909 q10: 1.8035 q20: 1.8848 q30: 1.9370 q40: 1.9730 q50: 2.0035 q60: 2.0287 q70: 2.0532 q80: 2.0725 q90: 2.0980 q100: 2.1942\n",
  3738. "[one_class_1 CSI 0.5939] [one_class_1 best 0.5939] \n",
  3739. "[one_class_mean CSI 0.5939] [one_class_mean best 0.5939] \n",
  3740. "0.5939\t0.5939\n"
  3741. ]
  3742. }
  3743. ],
  3744. "source": [
  3745. "# EVALUATION\n",
  3746. "# dataset : CNMC\n",
  3747. "# res : 450px\n",
  3748. "# id_class : all\n",
  3749. "# epoch : 100\n",
  3750. "# shift_tr : blur\n",
  3751. "# crop : 0.3\n",
  3752. "# blur_sigma : 2\n",
  3753. "# color_dist : 1\n",
  3754. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.3 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.3_color_dist1.0_one_class_0/last.model\""
  3755. ]
  3756. },
  3757. {
  3758. "cell_type": "code",
  3759. "execution_count": 210,
  3760. "id": "80437a6c",
  3761. "metadata": {
  3762. "scrolled": true
  3763. },
  3764. "outputs": [
  3765. {
  3766. "name": "stdout",
  3767. "output_type": "stream",
  3768. "text": [
  3769. "Pre-compute global statistics...\n",
  3770. "axis size: 3527 3527 3527 3527\n",
  3771. "weight_sim:\t0.0086\t0.0090\t0.0096\t0.0084\n",
  3772. "weight_shi:\t-0.6178\t0.6564\t1.4537\t1.9758\n",
  3773. "Pre-compute features...\n",
  3774. "Compute OOD scores... (score: CSI)\n",
  3775. "One_class_real_mean: 0.40624398667193307\n",
  3776. "CNMC 1.9234 +- 0.1679 q0: 1.2990 q10: 1.6940 q20: 1.7902 q30: 1.8538 q40: 1.9027 q50: 1.9438 q60: 1.9742 q70: 2.0088 q80: 2.0543 q90: 2.1136 q100: 2.6046\n",
  3777. "one_class_1 1.9913 +- 0.2119 q0: 1.3411 q10: 1.7247 q20: 1.8161 q30: 1.8739 q40: 1.9369 q50: 1.9987 q60: 2.0415 q70: 2.0979 q80: 2.1553 q90: 2.2420 q100: 2.6629\n",
  3778. "[one_class_1 CSI 0.4062] [one_class_1 best 0.4062] \n",
  3779. "[one_class_mean CSI 0.4062] [one_class_mean best 0.4062] \n",
  3780. "0.4062\t0.4062\n"
  3781. ]
  3782. }
  3783. ],
  3784. "source": [
  3785. "# EVALUATION\n",
  3786. "# dataset : CNMC\n",
  3787. "# res : 450px\n",
  3788. "# id_class : all\n",
  3789. "# epoch : 100\n",
  3790. "# shift_tr : blur\n",
  3791. "# crop : 0.02\n",
  3792. "# blur_sigma : 2\n",
  3793. "# color_dist : 1\n",
  3794. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.02 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.02_color_dist1.0_one_class_0/last.model\""
  3795. ]
  3796. },
  3797. {
  3798. "cell_type": "code",
  3799. "execution_count": 211,
  3800. "id": "5ee4b03d",
  3801. "metadata": {
  3802. "scrolled": true
  3803. },
  3804. "outputs": [
  3805. {
  3806. "name": "stdout",
  3807. "output_type": "stream",
  3808. "text": [
  3809. "Pre-compute global statistics...\n",
  3810. "axis size: 3527 3527 3527 3527\n",
  3811. "weight_sim:\t0.0077\t0.0063\t0.0079\t0.0085\n",
  3812. "weight_shi:\t-0.5622\t1.4395\t2.1736\t5.1802\n",
  3813. "Pre-compute features...\n",
  3814. "Compute OOD scores... (score: CSI)\n",
  3815. "One_class_real_mean: 0.40242330791278014\n",
  3816. "CNMC 1.7715 +- 0.3123 q0: 1.0132 q10: 1.4352 q20: 1.5132 q30: 1.5847 q40: 1.6416 q50: 1.7219 q60: 1.7995 q70: 1.8923 q80: 2.0104 q90: 2.1944 q100: 3.1272\n",
  3817. "one_class_1 1.9377 +- 0.4535 q0: 1.0669 q10: 1.4215 q20: 1.5153 q30: 1.6260 q40: 1.7391 q50: 1.8745 q60: 1.9976 q70: 2.1337 q80: 2.2999 q90: 2.5968 q100: 3.2364\n",
  3818. "[one_class_1 CSI 0.4024] [one_class_1 best 0.4024] \n",
  3819. "[one_class_mean CSI 0.4024] [one_class_mean best 0.4024] \n",
  3820. "0.4024\t0.4024\n"
  3821. ]
  3822. }
  3823. ],
  3824. "source": [
  3825. "# EVALUATION\n",
  3826. "# dataset : CNMC\n",
  3827. "# res : 450px\n",
  3828. "# id_class : all\n",
  3829. "# epoch : 100\n",
  3830. "# shift_tr : blur\n",
  3831. "# crop : 0.008\n",
  3832. "# blur_sigma : 2\n",
  3833. "# color_dist : 1\n",
  3834. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.008 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.008_color_dist1.0_one_class_0/last.model\""
  3835. ]
  3836. },
  3837. {
  3838. "cell_type": "markdown",
  3839. "id": "3993fc92",
  3840. "metadata": {},
  3841. "source": [
  3842. "## Examine blur_sigma"
  3843. ]
  3844. },
  3845. {
  3846. "cell_type": "code",
  3847. "execution_count": 212,
  3848. "id": "d11c9dcd",
  3849. "metadata": {
  3850. "scrolled": true
  3851. },
  3852. "outputs": [
  3853. {
  3854. "name": "stdout",
  3855. "output_type": "stream",
  3856. "text": [
  3857. "Pre-compute global statistics...\n",
  3858. "axis size: 3527 3527 3527 3527\n",
  3859. "weight_sim:\t0.0118\t0.0082\t0.0118\t0.0109\n",
  3860. "weight_shi:\t-0.5332\t0.3382\t1.2635\t1.1178\n",
  3861. "Pre-compute features...\n",
  3862. "Compute OOD scores... (score: CSI)\n",
  3863. "One_class_real_mean: 0.513900282563121\n",
  3864. "CNMC 1.8224 +- 0.5573 q0: 0.4695 q10: 1.0940 q20: 1.3195 q30: 1.5008 q40: 1.6572 q50: 1.8032 q60: 1.9693 q70: 2.1209 q80: 2.3099 q90: 2.5687 q100: 3.4860\n",
  3865. "one_class_1 1.8135 +- 0.7140 q0: 0.2666 q10: 0.8849 q20: 1.1651 q30: 1.3949 q40: 1.5485 q50: 1.7703 q60: 1.9382 q70: 2.1803 q80: 2.4559 q90: 2.7728 q100: 4.0059\n",
  3866. "[one_class_1 CSI 0.5139] [one_class_1 best 0.5139] \n",
  3867. "[one_class_mean CSI 0.5139] [one_class_mean best 0.5139] \n",
  3868. "0.5139\t0.5139\n"
  3869. ]
  3870. }
  3871. ],
  3872. "source": [
  3873. "# EVALUATION\n",
  3874. "# dataset : CNMC\n",
  3875. "# res : 450px\n",
  3876. "# id_class : all\n",
  3877. "# epoch : 100\n",
  3878. "# shift_tr : blur\n",
  3879. "# crop : 0.08\n",
  3880. "# blur_sigma : 40\n",
  3881. "# color_dist : 1\n",
  3882. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 40 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_40.0_resize_factor_0.08_color_dist1.0_one_class_0/last.model\""
  3883. ]
  3884. },
  3885. {
  3886. "cell_type": "code",
  3887. "execution_count": 213,
  3888. "id": "b5ffde5e",
  3889. "metadata": {
  3890. "scrolled": true
  3891. },
  3892. "outputs": [
  3893. {
  3894. "name": "stdout",
  3895. "output_type": "stream",
  3896. "text": [
  3897. "Pre-compute global statistics...\n",
  3898. "axis size: 3527 3527 3527 3527\n",
  3899. "weight_sim:\t0.0083\t0.0108\t0.0081\t0.0091\n",
  3900. "weight_shi:\t-0.0827\t0.1462\t0.2242\t0.2133\n",
  3901. "Pre-compute features...\n",
  3902. "Compute OOD scores... (score: CSI)\n",
  3903. "One_class_real_mean: 0.5305926482950001\n",
  3904. "CNMC 1.9788 +- 0.0509 q0: 1.8514 q10: 1.9121 q20: 1.9387 q30: 1.9537 q40: 1.9657 q50: 1.9801 q60: 1.9904 q70: 2.0027 q80: 2.0193 q90: 2.0412 q100: 2.1780\n",
  3905. "one_class_1 1.9783 +- 0.0729 q0: 1.8180 q10: 1.8943 q20: 1.9205 q30: 1.9377 q40: 1.9541 q50: 1.9673 q60: 1.9856 q70: 2.0070 q80: 2.0322 q90: 2.0712 q100: 2.2575\n",
  3906. "[one_class_1 CSI 0.5306] [one_class_1 best 0.5306] \n",
  3907. "[one_class_mean CSI 0.5306] [one_class_mean best 0.5306] \n",
  3908. "0.5306\t0.5306\n"
  3909. ]
  3910. }
  3911. ],
  3912. "source": [
  3913. "# EVALUATION\n",
  3914. "# dataset : CNMC\n",
  3915. "# res : 450px\n",
  3916. "# id_class : all\n",
  3917. "# epoch : 100\n",
  3918. "# shift_tr : blur\n",
  3919. "# crop : 0.08\n",
  3920. "# blur_sigma : 20\n",
  3921. "# color_dist : 1\n",
  3922. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 20 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_20.0_resize_factor_0.08_color_dist1.0_one_class_0/last.model\""
  3923. ]
  3924. },
  3925. {
  3926. "cell_type": "code",
  3927. "execution_count": 214,
  3928. "id": "46c0a5be",
  3929. "metadata": {
  3930. "scrolled": true
  3931. },
  3932. "outputs": [
  3933. {
  3934. "name": "stdout",
  3935. "output_type": "stream",
  3936. "text": [
  3937. "Pre-compute global statistics...\n",
  3938. "axis size: 3527 3527 3527 3527\n",
  3939. "weight_sim:\t0.0121\t0.0090\t0.0108\t0.0115\n",
  3940. "weight_shi:\t-0.1191\t0.1866\t0.3505\t0.3025\n",
  3941. "Pre-compute features...\n",
  3942. "Compute OOD scores... (score: CSI)\n",
  3943. "One_class_real_mean: 0.5126887552031113\n",
  3944. "CNMC 1.9350 +- 0.1456 q0: 1.5801 q10: 1.7447 q20: 1.8072 q30: 1.8557 q40: 1.8955 q50: 1.9330 q60: 1.9678 q70: 2.0099 q80: 2.0570 q90: 2.1285 q100: 2.4351\n",
  3945. "one_class_1 1.9410 +- 0.1925 q0: 1.5534 q10: 1.7034 q20: 1.7718 q30: 1.8241 q40: 1.8672 q50: 1.9158 q60: 1.9602 q70: 2.0200 q80: 2.1069 q90: 2.2114 q100: 2.5473\n",
  3946. "[one_class_1 CSI 0.5127] [one_class_1 best 0.5127] \n",
  3947. "[one_class_mean CSI 0.5127] [one_class_mean best 0.5127] \n",
  3948. "0.5127\t0.5127\n"
  3949. ]
  3950. }
  3951. ],
  3952. "source": [
  3953. "# EVALUATION\n",
  3954. "# dataset : CNMC\n",
  3955. "# res : 450px\n",
  3956. "# id_class : all\n",
  3957. "# epoch : 100\n",
  3958. "# shift_tr : blur\n",
  3959. "# crop : 0.08\n",
  3960. "# blur_sigma : 6\n",
  3961. "# color_dist : 1\n",
  3962. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 6 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_6.0_resize_factor_0.08_color_dist1.0_one_class_0/last.model\""
  3963. ]
  3964. },
  3965. {
  3966. "cell_type": "code",
  3967. "execution_count": 215,
  3968. "id": "9c074889",
  3969. "metadata": {
  3970. "scrolled": true
  3971. },
  3972. "outputs": [
  3973. {
  3974. "name": "stdout",
  3975. "output_type": "stream",
  3976. "text": [
  3977. "Pre-compute global statistics...\n",
  3978. "axis size: 3527 3527 3527 3527\n",
  3979. "weight_sim:\t0.0091\t0.0056\t0.0092\t0.0077\n",
  3980. "weight_shi:\t1.7473\t1.4099\t-3.2623\t7.2654\n",
  3981. "Pre-compute features...\n",
  3982. "Compute OOD scores... (score: CSI)\n",
  3983. "One_class_real_mean: 0.3547040683012791\n",
  3984. "CNMC 1.6471 +- 0.5245 q0: 0.2048 q10: 0.9772 q20: 1.1761 q30: 1.3840 q40: 1.5410 q50: 1.6358 q60: 1.7534 q70: 1.8764 q80: 2.0524 q90: 2.3159 q100: 3.4153\n",
  3985. "one_class_1 2.0065 +- 0.7329 q0: 0.2362 q10: 1.0671 q20: 1.3288 q30: 1.5682 q40: 1.7685 q50: 1.9697 q60: 2.1593 q70: 2.3883 q80: 2.6860 q90: 3.0258 q100: 4.4012\n",
  3986. "[one_class_1 CSI 0.3547] [one_class_1 best 0.3547] \n",
  3987. "[one_class_mean CSI 0.3547] [one_class_mean best 0.3547] \n",
  3988. "0.3547\t0.3547\n"
  3989. ]
  3990. }
  3991. ],
  3992. "source": [
  3993. "# EVALUATION\n",
  3994. "# dataset : CNMC\n",
  3995. "# res : 450px\n",
  3996. "# id_class : all\n",
  3997. "# epoch : 100\n",
  3998. "# shift_tr : blur\n",
  3999. "# crop : 0.08\n",
  4000. "# blur_sigma : 4\n",
  4001. "# color_dist : 1\n",
  4002. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 4 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist1.0_blur_sigma4.0_one_class_0/last.model\""
  4003. ]
  4004. },
  4005. {
  4006. "cell_type": "code",
  4007. "execution_count": 216,
  4008. "id": "99c14a28",
  4009. "metadata": {
  4010. "scrolled": true
  4011. },
  4012. "outputs": [
  4013. {
  4014. "name": "stdout",
  4015. "output_type": "stream",
  4016. "text": [
  4017. "Pre-compute global statistics...\n",
  4018. "axis size: 3527 3527 3527 3527\n",
  4019. "weight_sim:\t0.0033\t0.0033\t0.0025\t0.0033\n",
  4020. "weight_shi:\t0.2828\t-1.2986\t-0.7648\t-1.3398\n",
  4021. "Pre-compute features...\n",
  4022. "Compute OOD scores... (score: CSI)\n",
  4023. "One_class_real_mean: 0.5759223812272759\n",
  4024. "CNMC 1.9848 +- 0.2270 q0: 1.1862 q10: 1.7005 q20: 1.7975 q30: 1.8756 q40: 1.9360 q50: 1.9962 q60: 2.0534 q70: 2.1141 q80: 2.1839 q90: 2.2699 q100: 2.5657\n",
  4025. "one_class_1 1.9048 +- 0.2961 q0: 0.9850 q10: 1.4973 q20: 1.6832 q30: 1.7788 q40: 1.8554 q50: 1.9257 q60: 1.9946 q70: 2.0781 q80: 2.1586 q90: 2.2805 q100: 2.6712\n",
  4026. "[one_class_1 CSI 0.5759] [one_class_1 best 0.5759] \n",
  4027. "[one_class_mean CSI 0.5759] [one_class_mean best 0.5759] \n",
  4028. "0.5759\t0.5759\n"
  4029. ]
  4030. }
  4031. ],
  4032. "source": [
  4033. "# EVALUATION\n",
  4034. "# dataset : CNMC\n",
  4035. "# res : 450px\n",
  4036. "# id_class : all\n",
  4037. "# epoch : 100\n",
  4038. "# shift_tr : blur\n",
  4039. "# crop : 0.08\n",
  4040. "# blur_sigma : 3\n",
  4041. "# color_dist : 1\n",
  4042. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 3 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist1.0_blur_sigma3.0_one_class_0/last.model\""
  4043. ]
  4044. },
  4045. {
  4046. "cell_type": "code",
  4047. "execution_count": 221,
  4048. "id": "bd3e218a",
  4049. "metadata": {
  4050. "scrolled": true
  4051. },
  4052. "outputs": [
  4053. {
  4054. "name": "stdout",
  4055. "output_type": "stream",
  4056. "text": [
  4057. "Pre-compute global statistics...\n",
  4058. "axis size: 3527 3527 3527 3527\n",
  4059. "weight_sim:\t0.0050\t0.0049\t0.0049\t0.0050\n",
  4060. "weight_shi:\t0.3094\t-1.0241\t-0.9471\t-0.9535\n",
  4061. "Pre-compute features...\n",
  4062. "Compute OOD scores... (score: CSI)\n",
  4063. "One_class_real_mean: 0.5930793556750625\n",
  4064. "CNMC 2.0015 +- 0.1074 q0: 1.5164 q10: 1.8705 q20: 1.9260 q30: 1.9607 q40: 1.9870 q50: 2.0109 q60: 2.0342 q70: 2.0588 q80: 2.0868 q90: 2.1264 q100: 2.2720\n",
  4065. "one_class_1 1.9465 +- 0.1678 q0: 1.2629 q10: 1.7271 q20: 1.8484 q30: 1.9020 q40: 1.9374 q50: 1.9751 q60: 2.0018 q70: 2.0397 q80: 2.0761 q90: 2.1296 q100: 2.2873\n",
  4066. "[one_class_1 CSI 0.5931] [one_class_1 best 0.5931] \n",
  4067. "[one_class_mean CSI 0.5931] [one_class_mean best 0.5931] \n",
  4068. "0.5931\t0.5931\n"
  4069. ]
  4070. }
  4071. ],
  4072. "source": [
  4073. "# EVALUATION\n",
  4074. "# dataset : CNMC\n",
  4075. "# res : 450px\n",
  4076. "# id_class : all\n",
  4077. "# epoch : 100\n",
  4078. "# shift_tr : blur\n",
  4079. "# crop : 0.08\n",
  4080. "# blur_sigma : 2\n",
  4081. "# color_dist : 1\n",
  4082. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.08_color_dist1.0_one_class_0/last.model\""
  4083. ]
  4084. },
  4085. {
  4086. "cell_type": "code",
  4087. "execution_count": 218,
  4088. "id": "c2f0113b",
  4089. "metadata": {
  4090. "scrolled": true
  4091. },
  4092. "outputs": [
  4093. {
  4094. "name": "stdout",
  4095. "output_type": "stream",
  4096. "text": [
  4097. "Pre-compute global statistics...\n",
  4098. "axis size: 3527 3527 3527 3527\n",
  4099. "weight_sim:\t0.0151\t0.0114\t0.0140\t0.0143\n",
  4100. "weight_shi:\t0.3904\t-1.7955\t-0.8990\t-1.3060\n",
  4101. "Pre-compute features...\n",
  4102. "Compute OOD scores... (score: CSI)\n",
  4103. "One_class_real_mean: 0.6319476093539534\n",
  4104. "CNMC 2.0984 +- 0.1571 q0: 1.7583 q10: 1.9064 q20: 1.9536 q30: 2.0042 q40: 2.0398 q50: 2.0803 q60: 2.1205 q70: 2.1755 q80: 2.2444 q90: 2.3124 q100: 2.6504\n",
  4105. "one_class_1 2.0194 +- 0.1919 q0: 1.5830 q10: 1.7904 q20: 1.8384 q30: 1.8841 q40: 1.9493 q50: 1.9958 q60: 2.0566 q70: 2.1204 q80: 2.1982 q90: 2.2910 q100: 2.6254\n",
  4106. "[one_class_1 CSI 0.6319] [one_class_1 best 0.6319] \n",
  4107. "[one_class_mean CSI 0.6319] [one_class_mean best 0.6319] \n",
  4108. "0.6319\t0.6319\n"
  4109. ]
  4110. }
  4111. ],
  4112. "source": [
  4113. "# EVALUATION\n",
  4114. "# dataset : CNMC\n",
  4115. "# res : 450px\n",
  4116. "# id_class : all\n",
  4117. "# epoch : 100\n",
  4118. "# shift_tr : blur\n",
  4119. "# crop : 0.08\n",
  4120. "# blur_sigma : 1.5\n",
  4121. "# color_dist : 1\n",
  4122. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 1.5 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist1.0_blur_sigma1.5_one_class_0/last.model\""
  4123. ]
  4124. },
  4125. {
  4126. "cell_type": "code",
  4127. "execution_count": 219,
  4128. "id": "1a64397f",
  4129. "metadata": {
  4130. "scrolled": true
  4131. },
  4132. "outputs": [
  4133. {
  4134. "name": "stdout",
  4135. "output_type": "stream",
  4136. "text": [
  4137. "Pre-compute global statistics...\n",
  4138. "axis size: 3527 3527 3527 3527\n",
  4139. "weight_sim:\t0.0835\t0.0834\t0.0843\t0.0839\n",
  4140. "weight_shi:\t0.6194\t-4.8322\t-1.4623\t-2.0319\n",
  4141. "Pre-compute features...\n",
  4142. "Compute OOD scores... (score: CSI)\n",
  4143. "One_class_real_mean: 0.5156447806844306\n",
  4144. "CNMC 2.0671 +- 0.1604 q0: 1.6242 q10: 1.8621 q20: 1.9293 q30: 1.9753 q40: 2.0244 q50: 2.0641 q60: 2.0983 q70: 2.1559 q80: 2.2074 q90: 2.2798 q100: 2.5007\n",
  4145. "one_class_1 2.0520 +- 0.2129 q0: 1.5274 q10: 1.7652 q20: 1.8656 q30: 1.9325 q40: 1.9996 q50: 2.0618 q60: 2.1083 q70: 2.1784 q80: 2.2379 q90: 2.3346 q100: 2.6253\n",
  4146. "[one_class_1 CSI 0.5156] [one_class_1 best 0.5156] \n",
  4147. "[one_class_mean CSI 0.5156] [one_class_mean best 0.5156] \n",
  4148. "0.5156\t0.5156\n"
  4149. ]
  4150. }
  4151. ],
  4152. "source": [
  4153. "# EVALUATION\n",
  4154. "# dataset : CNMC\n",
  4155. "# res : 450px\n",
  4156. "# id_class : all\n",
  4157. "# epoch : 100\n",
  4158. "# shift_tr : blur\n",
  4159. "# crop : 0.08\n",
  4160. "# blur_sigma : 1\n",
  4161. "# color_dist : 1\n",
  4162. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 1 --resize_factor 0.08 --blur_sigma 1 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist1.0/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist1.0_blur_sigma1.0_one_class_0/last.model\""
  4163. ]
  4164. },
  4165. {
  4166. "cell_type": "markdown",
  4167. "id": "c1bce058",
  4168. "metadata": {},
  4169. "source": [
  4170. "# Color Distortion = 0.5"
  4171. ]
  4172. },
  4173. {
  4174. "cell_type": "markdown",
  4175. "id": "65e662af",
  4176. "metadata": {},
  4177. "source": [
  4178. "## Examine crop"
  4179. ]
  4180. },
  4181. {
  4182. "cell_type": "code",
  4183. "execution_count": 184,
  4184. "id": "fdaec3de",
  4185. "metadata": {
  4186. "scrolled": true
  4187. },
  4188. "outputs": [
  4189. {
  4190. "name": "stdout",
  4191. "output_type": "stream",
  4192. "text": [
  4193. "Pre-compute global statistics...\n",
  4194. "axis size: 3527 3527 3527 3527\n",
  4195. "weight_sim:\t0.0077\t0.0072\t0.0077\t0.0083\n",
  4196. "weight_shi:\t-0.2495\t0.5029\t0.4407\t0.6284\n",
  4197. "Pre-compute features...\n",
  4198. "Compute OOD scores... (score: CSI)\n",
  4199. "One_class_real_mean: 0.4545575962892069\n",
  4200. "CNMC 1.9531 +- 0.0857 q0: 1.5656 q10: 1.8568 q20: 1.8889 q30: 1.9090 q40: 1.9290 q50: 1.9470 q60: 1.9664 q70: 1.9868 q80: 2.0131 q90: 2.0534 q100: 2.4858\n",
  4201. "one_class_1 1.9770 +- 0.1276 q0: 1.5910 q10: 1.8422 q20: 1.8816 q30: 1.9115 q40: 1.9369 q50: 1.9621 q60: 1.9818 q70: 2.0174 q80: 2.0584 q90: 2.1323 q100: 2.7000\n",
  4202. "[one_class_1 CSI 0.4546] [one_class_1 best 0.4546] \n",
  4203. "[one_class_mean CSI 0.4546] [one_class_mean best 0.4546] \n",
  4204. "0.4546\t0.4546\n"
  4205. ]
  4206. }
  4207. ],
  4208. "source": [
  4209. "# EVALUATION\n",
  4210. "# dataset : CNMC\n",
  4211. "# res : 450px\n",
  4212. "# id_class : all\n",
  4213. "# epoch : 100\n",
  4214. "# shift_tr : blur\n",
  4215. "# crop : 0.5\n",
  4216. "# blur_sigma : 2\n",
  4217. "# color_dist : 0.5\n",
  4218. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.5 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.5_color_dist0.5_one_class_0/last.model\""
  4219. ]
  4220. },
  4221. {
  4222. "cell_type": "code",
  4223. "execution_count": 185,
  4224. "id": "eaa5ec79",
  4225. "metadata": {
  4226. "scrolled": true
  4227. },
  4228. "outputs": [
  4229. {
  4230. "name": "stdout",
  4231. "output_type": "stream",
  4232. "text": [
  4233. "Pre-compute global statistics...\n",
  4234. "axis size: 3527 3527 3527 3527\n",
  4235. "weight_sim:\t0.0069\t0.0088\t0.0090\t0.0079\n",
  4236. "weight_shi:\t2.7516\t0.9415\t1.1553\t-18.6953\n",
  4237. "Pre-compute features...\n",
  4238. "Compute OOD scores... (score: CSI)\n",
  4239. "One_class_real_mean: 0.580519095798013\n",
  4240. "CNMC 2.6811 +- 1.0535 q0: -1.5616 q10: 1.2724 q20: 1.7695 q30: 2.1545 q40: 2.4883 q50: 2.7551 q60: 3.0169 q70: 3.2695 q80: 3.5643 q90: 3.8850 q100: 6.2124\n",
  4241. "one_class_1 2.2993 +- 1.4215 q0: -2.7435 q10: 0.4967 q20: 1.2345 q30: 1.7164 q40: 2.0762 q50: 2.3752 q60: 2.6957 q70: 3.0288 q80: 3.4597 q90: 3.9539 q100: 6.3139\n",
  4242. "[one_class_1 CSI 0.5805] [one_class_1 best 0.5805] \n",
  4243. "[one_class_mean CSI 0.5805] [one_class_mean best 0.5805] \n",
  4244. "0.5805\t0.5805\n"
  4245. ]
  4246. }
  4247. ],
  4248. "source": [
  4249. "# EVALUATION\n",
  4250. "# dataset : CNMC\n",
  4251. "# res : 450px\n",
  4252. "# id_class : all\n",
  4253. "# epoch : 100\n",
  4254. "# shift_tr : blur\n",
  4255. "# crop : 0.3\n",
  4256. "# blur_sigma : 2\n",
  4257. "# color_dist : 0.5\n",
  4258. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.3 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.3_color_dist0.5_one_class_0/last.model\""
  4259. ]
  4260. },
  4261. {
  4262. "cell_type": "code",
  4263. "execution_count": 220,
  4264. "id": "4a75f4d4",
  4265. "metadata": {
  4266. "scrolled": true
  4267. },
  4268. "outputs": [
  4269. {
  4270. "name": "stdout",
  4271. "output_type": "stream",
  4272. "text": [
  4273. "Pre-compute global statistics...\n",
  4274. "axis size: 3527 3527 3527 3527\n",
  4275. "weight_sim:\t0.0074\t0.0080\t0.0073\t0.0077\n",
  4276. "weight_shi:\t-0.8732\t0.8498\t2.4905\t1.5653\n",
  4277. "Pre-compute features...\n",
  4278. "Compute OOD scores... (score: CSI)\n",
  4279. "One_class_real_mean: 0.2671803947781525\n",
  4280. "CNMC 1.8416 +- 0.1772 q0: 1.2265 q10: 1.6388 q20: 1.7083 q30: 1.7532 q40: 1.7915 q50: 1.8242 q60: 1.8580 q70: 1.8990 q80: 1.9656 q90: 2.0775 q100: 2.5900\n",
  4281. "one_class_1 2.0431 +- 0.2719 q0: 1.2846 q10: 1.7272 q20: 1.8128 q30: 1.8857 q40: 1.9439 q50: 2.0013 q60: 2.0829 q70: 2.1644 q80: 2.2684 q90: 2.4103 q100: 2.9156\n",
  4282. "[one_class_1 CSI 0.2672] [one_class_1 best 0.2672] \n",
  4283. "[one_class_mean CSI 0.2672] [one_class_mean best 0.2672] \n",
  4284. "0.2672\t0.2672\n"
  4285. ]
  4286. }
  4287. ],
  4288. "source": [
  4289. "# EVALUATION\n",
  4290. "# dataset : CNMC\n",
  4291. "# res : 450px\n",
  4292. "# id_class : all\n",
  4293. "# epoch : 100\n",
  4294. "# shift_tr : blur\n",
  4295. "# crop : 0.02\n",
  4296. "# blur_sigma : 2\n",
  4297. "# color_dist : 0.5\n",
  4298. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.02 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.02_color_dist0.5_one_class_0/last.model\""
  4299. ]
  4300. },
  4301. {
  4302. "cell_type": "code",
  4303. "execution_count": 187,
  4304. "id": "9d31d62a",
  4305. "metadata": {
  4306. "scrolled": true
  4307. },
  4308. "outputs": [
  4309. {
  4310. "name": "stdout",
  4311. "output_type": "stream",
  4312. "text": [
  4313. "Pre-compute global statistics...\n",
  4314. "axis size: 3527 3527 3527 3527\n",
  4315. "weight_sim:\t0.0046\t0.0037\t0.0035\t0.0047\n",
  4316. "weight_shi:\t0.4014\t-0.7791\t-0.6536\t-1.3711\n",
  4317. "Pre-compute features...\n",
  4318. "Compute OOD scores... (score: CSI)\n",
  4319. "One_class_real_mean: 0.611233276618155\n",
  4320. "CNMC 1.9991 +- 0.2593 q0: 0.9291 q10: 1.6096 q20: 1.7794 q30: 1.9130 q40: 1.9994 q50: 2.0695 q60: 2.1203 q70: 2.1657 q80: 2.2124 q90: 2.2692 q100: 2.5149\n",
  4321. "one_class_1 1.8852 +- 0.3136 q0: 0.6811 q10: 1.4317 q20: 1.6563 q30: 1.7768 q40: 1.8804 q50: 1.9566 q60: 2.0179 q70: 2.0924 q80: 2.1459 q90: 2.2152 q100: 2.4864\n",
  4322. "[one_class_1 CSI 0.6112] [one_class_1 best 0.6112] \n",
  4323. "[one_class_mean CSI 0.6112] [one_class_mean best 0.6112] \n",
  4324. "0.6112\t0.6112\n"
  4325. ]
  4326. }
  4327. ],
  4328. "source": [
  4329. "# EVALUATION\n",
  4330. "# dataset : CNMC\n",
  4331. "# res : 450px\n",
  4332. "# id_class : all\n",
  4333. "# epoch : 100\n",
  4334. "# shift_tr : blur\n",
  4335. "# crop : 0.008\n",
  4336. "# blur_sigma : 2\n",
  4337. "# color_dist : 0.5\n",
  4338. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.008 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.008_color_dist0.5_one_class_0/last.model\""
  4339. ]
  4340. },
  4341. {
  4342. "cell_type": "markdown",
  4343. "id": "58a14458",
  4344. "metadata": {},
  4345. "source": [
  4346. "## Examine blur_sigma"
  4347. ]
  4348. },
  4349. {
  4350. "cell_type": "code",
  4351. "execution_count": 188,
  4352. "id": "c7c2318d",
  4353. "metadata": {
  4354. "scrolled": true
  4355. },
  4356. "outputs": [
  4357. {
  4358. "name": "stdout",
  4359. "output_type": "stream",
  4360. "text": [
  4361. "Pre-compute global statistics...\n",
  4362. "axis size: 3527 3527 3527 3527\n",
  4363. "weight_sim:\t0.0050\t0.0073\t0.0050\t0.0055\n",
  4364. "weight_shi:\t-0.3869\t0.3100\t0.7499\t0.9321\n",
  4365. "Pre-compute features...\n",
  4366. "Compute OOD scores... (score: CSI)\n",
  4367. "One_class_real_mean: 0.4740262206422994\n",
  4368. "CNMC 1.9842 +- 0.2291 q0: 1.4232 q10: 1.6950 q20: 1.7851 q30: 1.8500 q40: 1.9214 q50: 1.9744 q60: 2.0301 q70: 2.0950 q80: 2.1712 q90: 2.2769 q100: 3.0240\n",
  4369. "one_class_1 2.0169 +- 0.2738 q0: 1.4504 q10: 1.6924 q20: 1.7765 q30: 1.8481 q40: 1.9251 q50: 1.9917 q60: 2.0673 q70: 2.1457 q80: 2.2342 q90: 2.3550 q100: 3.2798\n",
  4370. "[one_class_1 CSI 0.4740] [one_class_1 best 0.4740] \n",
  4371. "[one_class_mean CSI 0.4740] [one_class_mean best 0.4740] \n",
  4372. "0.4740\t0.4740\n"
  4373. ]
  4374. }
  4375. ],
  4376. "source": [
  4377. "# EVALUATION\n",
  4378. "# dataset : CNMC\n",
  4379. "# res : 450px\n",
  4380. "# id_class : all\n",
  4381. "# epoch : 100\n",
  4382. "# shift_tr : blur\n",
  4383. "# crop : 0.08\n",
  4384. "# blur_sigma : 40\n",
  4385. "# color_dist : 0.5\n",
  4386. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 40 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_40.0_resize_factor_0.08_color_dist0.5_one_class_0/last.model\""
  4387. ]
  4388. },
  4389. {
  4390. "cell_type": "code",
  4391. "execution_count": 189,
  4392. "id": "dbd4fb10",
  4393. "metadata": {
  4394. "scrolled": true
  4395. },
  4396. "outputs": [
  4397. {
  4398. "name": "stdout",
  4399. "output_type": "stream",
  4400. "text": [
  4401. "Pre-compute global statistics...\n",
  4402. "axis size: 3527 3527 3527 3527\n",
  4403. "weight_sim:\t0.0041\t0.0073\t0.0038\t0.0040\n",
  4404. "weight_shi:\t-0.0807\t0.1383\t0.2679\t0.2225\n",
  4405. "Pre-compute features...\n",
  4406. "Compute OOD scores... (score: CSI)\n",
  4407. "One_class_real_mean: 0.3159839323874052\n",
  4408. "CNMC 1.9782 +- 0.0390 q0: 1.8641 q10: 1.9344 q20: 1.9491 q30: 1.9576 q40: 1.9654 q50: 1.9735 q60: 1.9830 q70: 1.9942 q80: 2.0052 q90: 2.0239 q100: 2.1760\n",
  4409. "one_class_1 2.0111 +- 0.0558 q0: 1.8790 q10: 1.9491 q20: 1.9646 q30: 1.9780 q40: 1.9912 q50: 2.0041 q60: 2.0170 q70: 2.0318 q80: 2.0532 q90: 2.0897 q100: 2.2666\n",
  4410. "[one_class_1 CSI 0.3160] [one_class_1 best 0.3160] \n",
  4411. "[one_class_mean CSI 0.3160] [one_class_mean best 0.3160] \n",
  4412. "0.3160\t0.3160\n"
  4413. ]
  4414. }
  4415. ],
  4416. "source": [
  4417. "# EVALUATION\n",
  4418. "# dataset : CNMC\n",
  4419. "# res : 450px\n",
  4420. "# id_class : all\n",
  4421. "# epoch : 100\n",
  4422. "# shift_tr : blur\n",
  4423. "# crop : 0.08\n",
  4424. "# blur_sigma : 20\n",
  4425. "# color_dist : 0.5\n",
  4426. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 20 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_20.0_resize_factor_0.08_color_dist0.5_one_class_0/last.model\""
  4427. ]
  4428. },
  4429. {
  4430. "cell_type": "code",
  4431. "execution_count": 190,
  4432. "id": "c0cd8374",
  4433. "metadata": {
  4434. "scrolled": true
  4435. },
  4436. "outputs": [
  4437. {
  4438. "name": "stdout",
  4439. "output_type": "stream",
  4440. "text": [
  4441. "Pre-compute global statistics...\n",
  4442. "axis size: 3527 3527 3527 3527\n",
  4443. "weight_sim:\t0.0021\t0.0037\t0.0024\t0.0027\n",
  4444. "weight_shi:\t0.1478\t4.1795\t-0.4613\t-0.5806\n",
  4445. "Pre-compute features...\n",
  4446. "Compute OOD scores... (score: CSI)\n",
  4447. "One_class_real_mean: 0.4508957959874011\n",
  4448. "CNMC 2.0731 +- 0.5687 q0: 0.4702 q10: 1.3853 q20: 1.5945 q30: 1.7650 q40: 1.9267 q50: 2.0493 q60: 2.1848 q70: 2.3330 q80: 2.5050 q90: 2.7946 q100: 4.6939\n",
  4449. "one_class_1 2.1855 +- 0.7534 q0: 0.3032 q10: 1.1734 q20: 1.4954 q30: 1.7768 q40: 1.9835 q50: 2.1717 q60: 2.4165 q70: 2.5852 q80: 2.8103 q90: 3.1495 q100: 4.4871\n",
  4450. "[one_class_1 CSI 0.4509] [one_class_1 best 0.4509] \n",
  4451. "[one_class_mean CSI 0.4509] [one_class_mean best 0.4509] \n",
  4452. "0.4509\t0.4509\n"
  4453. ]
  4454. }
  4455. ],
  4456. "source": [
  4457. "# EVALUATION\n",
  4458. "# dataset : CNMC\n",
  4459. "# res : 450px\n",
  4460. "# id_class : all\n",
  4461. "# epoch : 100\n",
  4462. "# shift_tr : blur\n",
  4463. "# crop : 0.08\n",
  4464. "# blur_sigma : 6\n",
  4465. "# color_dist : 0.5\n",
  4466. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 6 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_6.0_resize_factor_0.08_color_dist0.5_one_class_0/last.model\""
  4467. ]
  4468. },
  4469. {
  4470. "cell_type": "code",
  4471. "execution_count": 191,
  4472. "id": "1a733a07",
  4473. "metadata": {
  4474. "scrolled": true
  4475. },
  4476. "outputs": [
  4477. {
  4478. "name": "stdout",
  4479. "output_type": "stream",
  4480. "text": [
  4481. "Pre-compute global statistics...\n",
  4482. "axis size: 3527 3527 3527 3527\n",
  4483. "weight_sim:\t0.0019\t0.0025\t0.0018\t0.0018\n",
  4484. "weight_shi:\t0.1207\t-0.4216\t-0.2927\t-0.2699\n",
  4485. "Pre-compute features...\n",
  4486. "Compute OOD scores... (score: CSI)\n",
  4487. "One_class_real_mean: 0.622416167876928\n",
  4488. "CNMC 2.0481 +- 0.2777 q0: 0.5649 q10: 1.7109 q20: 1.8569 q30: 1.9499 q40: 2.0216 q50: 2.0813 q60: 2.1374 q70: 2.2010 q80: 2.2718 q90: 2.3476 q100: 2.6884\n",
  4489. "one_class_1 1.8936 +- 0.3857 q0: 0.4436 q10: 1.4038 q20: 1.6226 q30: 1.7768 q40: 1.8682 q50: 1.9483 q60: 2.0252 q70: 2.1209 q80: 2.2012 q90: 2.3215 q100: 2.8144\n",
  4490. "[one_class_1 CSI 0.6224] [one_class_1 best 0.6224] \n",
  4491. "[one_class_mean CSI 0.6224] [one_class_mean best 0.6224] \n",
  4492. "0.6224\t0.6224\n"
  4493. ]
  4494. }
  4495. ],
  4496. "source": [
  4497. "# EVALUATION\n",
  4498. "# dataset : CNMC\n",
  4499. "# res : 450px\n",
  4500. "# id_class : all\n",
  4501. "# epoch : 100\n",
  4502. "# shift_tr : blur\n",
  4503. "# crop : 0.08\n",
  4504. "# blur_sigma : 4\n",
  4505. "# color_dist : 0.5\n",
  4506. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 4 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma4.0_one_class_0/last.model\""
  4507. ]
  4508. },
  4509. {
  4510. "cell_type": "code",
  4511. "execution_count": 192,
  4512. "id": "c59e2e1d",
  4513. "metadata": {
  4514. "scrolled": true
  4515. },
  4516. "outputs": [
  4517. {
  4518. "name": "stdout",
  4519. "output_type": "stream",
  4520. "text": [
  4521. "Pre-compute global statistics...\n",
  4522. "axis size: 3527 3527 3527 3527\n",
  4523. "weight_sim:\t0.0024\t0.0049\t0.0029\t0.0029\n",
  4524. "weight_shi:\t0.3727\t0.6016\t-2.1896\t-1.0076\n",
  4525. "Pre-compute features...\n",
  4526. "Compute OOD scores... (score: CSI)\n",
  4527. "One_class_real_mean: 0.7071838381996982\n",
  4528. "CNMC 2.1791 +- 0.2772 q0: 0.2329 q10: 1.8709 q20: 2.0154 q30: 2.0976 q40: 2.1641 q50: 2.2225 q60: 2.2692 q70: 2.3190 q80: 2.3739 q90: 2.4494 q100: 2.9055\n",
  4529. "one_class_1 1.9359 +- 0.4103 q0: -0.1517 q10: 1.4452 q20: 1.7034 q30: 1.8312 q40: 1.9334 q50: 2.0115 q60: 2.0642 q70: 2.1519 q80: 2.2408 q90: 2.3584 q100: 2.8261\n",
  4530. "[one_class_1 CSI 0.7072] [one_class_1 best 0.7072] \n",
  4531. "[one_class_mean CSI 0.7072] [one_class_mean best 0.7072] \n",
  4532. "0.7072\t0.7072\n"
  4533. ]
  4534. }
  4535. ],
  4536. "source": [
  4537. "# EVALUATION\n",
  4538. "# dataset : CNMC\n",
  4539. "# res : 450px\n",
  4540. "# id_class : all\n",
  4541. "# epoch : 100\n",
  4542. "# shift_tr : blur\n",
  4543. "# crop : 0.08\n",
  4544. "# blur_sigma : 3\n",
  4545. "# color_dist : 0.5\n",
  4546. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 3 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma3.0_one_class_0/last.model\""
  4547. ]
  4548. },
  4549. {
  4550. "cell_type": "code",
  4551. "execution_count": 193,
  4552. "id": "5827615d",
  4553. "metadata": {
  4554. "scrolled": true
  4555. },
  4556. "outputs": [
  4557. {
  4558. "name": "stdout",
  4559. "output_type": "stream",
  4560. "text": [
  4561. "Pre-compute global statistics...\n",
  4562. "axis size: 3527 3527 3527 3527\n",
  4563. "weight_sim:\t0.0019\t0.0027\t0.0022\t0.0026\n",
  4564. "weight_shi:\t0.1899\t-0.4837\t-0.3535\t-0.3448\n",
  4565. "Pre-compute features...\n",
  4566. "Compute OOD scores... (score: CSI)\n",
  4567. "One_class_real_mean: 0.7415028509504856\n",
  4568. "CNMC 2.1337 +- 0.1823 q0: 0.9904 q10: 1.9206 q20: 2.0374 q30: 2.0868 q40: 2.1243 q50: 2.1600 q60: 2.1945 q70: 2.2226 q80: 2.2642 q90: 2.3222 q100: 2.5460\n",
  4569. "one_class_1 1.9400 +- 0.2874 q0: 0.6323 q10: 1.5817 q20: 1.7870 q30: 1.8866 q40: 1.9416 q50: 1.9843 q60: 2.0207 q70: 2.0829 q80: 2.1566 q90: 2.2468 q100: 2.5500\n",
  4570. "[one_class_1 CSI 0.7415] [one_class_1 best 0.7415] \n",
  4571. "[one_class_mean CSI 0.7415] [one_class_mean best 0.7415] \n",
  4572. "0.7415\t0.7415\n"
  4573. ]
  4574. }
  4575. ],
  4576. "source": [
  4577. "# EVALUATION\n",
  4578. "# dataset : CNMC\n",
  4579. "# res : 450px\n",
  4580. "# id_class : all\n",
  4581. "# epoch : 100\n",
  4582. "# shift_tr : blur\n",
  4583. "# crop : 0.08\n",
  4584. "# blur_sigma : 2\n",
  4585. "# color_dist : 0.5\n",
  4586. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 2 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_2.0_resize_factor_0.08_color_dist0.5_one_class_0_7415/last.model\""
  4587. ]
  4588. },
  4589. {
  4590. "cell_type": "code",
  4591. "execution_count": 194,
  4592. "id": "65baeab1",
  4593. "metadata": {
  4594. "scrolled": true
  4595. },
  4596. "outputs": [
  4597. {
  4598. "name": "stdout",
  4599. "output_type": "stream",
  4600. "text": [
  4601. "Pre-compute global statistics...\n",
  4602. "axis size: 3527 3527 3527 3527\n",
  4603. "weight_sim:\t0.0022\t0.0033\t0.0025\t0.0029\n",
  4604. "weight_shi:\t0.4059\t-6.1160\t-2.6702\t-1.5404\n",
  4605. "Pre-compute features...\n",
  4606. "Compute OOD scores... (score: CSI)\n",
  4607. "One_class_real_mean: 0.7402292913641013\n",
  4608. "CNMC 2.5607 +- 0.6482 q0: -0.8214 q10: 1.7859 q20: 2.1154 q30: 2.3402 q40: 2.5031 q50: 2.6399 q60: 2.7654 q70: 2.9084 q80: 3.0413 q90: 3.2796 q100: 4.2054\n",
  4609. "one_class_1 1.8328 +- 0.9715 q0: -2.1220 q10: 0.6263 q20: 1.0844 q30: 1.4277 q40: 1.6691 q50: 1.8643 q60: 2.1102 q70: 2.3723 q80: 2.6504 q90: 3.0382 q100: 4.2076\n",
  4610. "[one_class_1 CSI 0.7402] [one_class_1 best 0.7402] \n",
  4611. "[one_class_mean CSI 0.7402] [one_class_mean best 0.7402] \n",
  4612. "0.7402\t0.7402\n"
  4613. ]
  4614. }
  4615. ],
  4616. "source": [
  4617. "# EVALUATION\n",
  4618. "# dataset : CNMC\n",
  4619. "# res : 450px\n",
  4620. "# id_class : all\n",
  4621. "# epoch : 100\n",
  4622. "# shift_tr : blur\n",
  4623. "# crop : 0.08\n",
  4624. "# blur_sigma : 1.5\n",
  4625. "# color_dist : 0.5\n",
  4626. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 1.5 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma1.5_one_class_0/last.model\""
  4627. ]
  4628. },
  4629. {
  4630. "cell_type": "code",
  4631. "execution_count": 195,
  4632. "id": "a9c1c45f",
  4633. "metadata": {
  4634. "scrolled": true
  4635. },
  4636. "outputs": [
  4637. {
  4638. "name": "stdout",
  4639. "output_type": "stream",
  4640. "text": [
  4641. "Pre-compute global statistics...\n",
  4642. "axis size: 3527 3527 3527 3527\n",
  4643. "weight_sim:\t0.0077\t0.0092\t0.0076\t0.0081\n",
  4644. "weight_shi:\t-0.2259\t0.4304\t0.6270\t0.7623\n",
  4645. "Pre-compute features...\n",
  4646. "Compute OOD scores... (score: CSI)\n",
  4647. "One_class_real_mean: 0.36727255694305183\n",
  4648. "CNMC 1.9613 +- 0.2117 q0: 1.4830 q10: 1.7342 q20: 1.7989 q30: 1.8500 q40: 1.8941 q50: 1.9317 q60: 1.9759 q70: 2.0278 q80: 2.0943 q90: 2.2144 q100: 3.2689\n",
  4649. "one_class_1 2.0967 +- 0.3131 q0: 1.5468 q10: 1.7976 q20: 1.8621 q30: 1.9190 q40: 1.9608 q50: 2.0266 q60: 2.0831 q70: 2.1520 q80: 2.2654 q90: 2.5291 q100: 3.5407\n",
  4650. "[one_class_1 CSI 0.3673] [one_class_1 best 0.3673] \n",
  4651. "[one_class_mean CSI 0.3673] [one_class_mean best 0.3673] \n",
  4652. "0.3673\t0.3673\n"
  4653. ]
  4654. }
  4655. ],
  4656. "source": [
  4657. "# EVALUATION\n",
  4658. "# dataset : CNMC\n",
  4659. "# res : 450px\n",
  4660. "# id_class : all\n",
  4661. "# epoch : 100\n",
  4662. "# shift_tr : blur\n",
  4663. "# crop : 0.08\n",
  4664. "# blur_sigma : 1.0\n",
  4665. "# color_dist : 0.5\n",
  4666. "!CUDA_VISIBLE_DEVICES=0 python3 \"eval.py\" --color_distort 0.5 --resize_factor 0.08 --blur_sigma 1.0 --mode ood_pre --dataset CNMC --model resnet18_imagenet --ood_score CSI --shift_trans_type blur --print_score --save_score --ood_samples 10 --resize_fix --one_class_idx 0 --load_path \"logs/id_all/color_dist0.5/blur/CNMC_resnet18_imagenet_unsup_simclr_CSI_450px_shift_blur_resize_factor0.08_color_dist0.5_blur_sigma1.0_one_class_0/last.model\""
  4667. ]
  4668. }
  4669. ],
  4670. "metadata": {
  4671. "kernelspec": {
  4672. "display_name": "Python 3",
  4673. "language": "python",
  4674. "name": "python3"
  4675. },
  4676. "language_info": {
  4677. "codemirror_mode": {
  4678. "name": "ipython",
  4679. "version": 3
  4680. },
  4681. "file_extension": ".py",
  4682. "mimetype": "text/x-python",
  4683. "name": "python",
  4684. "nbconvert_exporter": "python",
  4685. "pygments_lexer": "ipython3",
  4686. "version": "3.6.9"
  4687. }
  4688. },
  4689. "nbformat": 4,
  4690. "nbformat_minor": 5
  4691. }