A Smart Cache Content Update Policy Based on Deep Reinforcement Learning

<div>The average latency vs. the varying Zipf parameters. We assume that the <span class="nowrap"><svg height="12.4764pt" id="M197" style="vertical-align:-3.1685pt" version="1.1" viewbox="-0.0498162 -9.3079 279.134 12.4764" width="279.134pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M561 176C543 127 527 95 506 73C478 43 441 36 316 36C247 36 185 38 153 42C290 245 427 439 566 636L559 650H160C130 650 119 652 107 675H83C81 630 76 555 71 492L100 491C112 535 123 560 135 578C152 605 168 614 276 614H449C310 411 169 213 29 15L37 0H554C565 41 583 134 590 170L561 176Z"></path></g><g transform="matrix(.013,0,0,-0.013,8.022,0)"><path d="M135 536C164 536 186 560 186 587C186 617 164 639 136 639C109 639 85 617 85 587C85 560 109 536 135 536ZM252 0V26C188 32 181 38 181 106V451C138 433 90 420 39 412V388C99 379 102 374 102 312V106C102 38 95 32 32 26V0H252Z"></path></g><g transform="matrix(.013,0,0,-0.013,11.349,0)"><path d="M169 380V459C122 440 66 423 24 416V392C86 384 90 382 90 317V-135C90 -201 81 -207 17 -213V-240H253V-213C176 -207 169 -201 169 -125V6C182 -1 208 -11 238 -12C368 12 487 109 487 260C487 358 421 449 310 449C298 449 279 444 261 433L169 380ZM169 346C196 367 237 389 269 389C341 389 403 329 403 221C403 109 347 37 263 37C228 37 191 53 169 76V346Z"></path></g><g transform="matrix(.013,0,0,-0.013,18.004,0)"><path d="M54 437L27 408L31 397H101V103C101 37 94 32 30 26V0H266V25C187 33 180 36 180 110V397H288C299 404 304 428 298 437H180V477C179 562 190 610 203 630C214 647 230 659 256 659C289 659 318 641 337 622C346 612 355 612 364 619C374 627 380 635 383 643C388 655 387 667 378 678C362 697 333 710 299 712C260 707 225 689 189 659C135 613 119 563 112 541S101 490 101 458V437H54Z"></path></g><g transform="matrix(.013,0,0,-0.013,24.11,0)"><path d="M169 380V459C122 440 66 423 24 416V392C86 384 90 382 90 317V-135C90 -201 81 -207 17 -213V-240H253V-213C176 -207 169 -201 169 -125V6C182 -1 208 -11 238 -12C368 12 487 109 487 260C487 358 421 449 310 449C298 449 279 444 261 433L169 380ZM169 346C196 367 237 389 269 389C341 389 403 329 403 221C403 109 347 37 263 37C228 37 191 53 169 76V346Z"></path></g><g transform="matrix(.013,0,0,-0.013,30.987,0)"><path d="M433 39L423 65C413 59 399 54 387 54C370 54 352 69 352 114V299C352 352 342 392 307 422C285 440 255 449 225 449C168 437 102 399 75 379C56 365 44 353 44 339C44 315 69 296 87 296C101 296 111 303 116 319C124 349 133 371 145 385C156 397 171 404 190 404C241 404 275 364 275 291V274C253 256 180 229 120 209C65 190 39 159 39 110C39 47 88 -12 159 -12C189 -12 237 25 277 52C282 35 288 21 301 8C312 -3 333 -12 348 -12L433 39ZM275 84C256 65 221 48 195 48C164 48 124 73 124 124C124 161 146 180 185 198C206 208 254 229 275 240V84Z"></path></g><g transform="matrix(.013,0,0,-0.013,36.577,0)"><path d="M181 342V451C133 431 89 419 40 411V388C98 381 102 377 102 311V104C102 38 95 32 33 26V0H263V26C186 32 181 38 181 104V287C203 343 235 372 261 372C277 372 289 366 304 352C310 346 318 345 330 350C349 359 362 379 362 399C362 422 338 449 304 449C256 449 213 393 183 342H181Z"></path></g><g transform="matrix(.013,0,0,-0.013,41.4,0)"><path d="M433 39L423 65C413 59 399 54 387 54C370 54 352 69 352 114V299C352 352 342 392 307 422C285 440 255 449 225 449C168 437 102 399 75 379C56 365 44 353 44 339C44 315 69 296 87 296C101 296 111 303 116 319C124 349 133 371 145 385C156 397 171 404 190 404C241 404 275 364 275 291V274C253 256 180 229 120 209C65 190 39 159 39 110C39 47 88 -12 159 -12C189 -12 237 25 277 52C282 35 288 21 301 8C312 -3 333 -12 348 -12L433 39ZM275 84C256 65 221 48 195 48C164 48 124 73 124 124C124 161 146 180 185 198C206 208 254 229 275 240V84Z"></path></g><g transform="matrix(.013,0,0,-0.013,46.99,0)"><path d="M797 0V26C739 32 732 36 732 103V296C732 394 682 449 605 449C576 449 550 437 529 423C504 407 475 389 446 366C425 418 382 449 334 449C303 449 279 437 253 421C222 403 201 385 180 371V452C135 432 85 419 41 411V388C99 379 102 374 102 310V103C102 38 93 32 27 26V0H238V26C189 32 180 38 180 103V338C210 363 250 390 289 390C351 390 377 348 377 275V103C377 37 368 32 306 26V0H520V26C465 32 456 38 456 101V296C456 314 455 326 453 338C491 369 529 390 565 390C628 390 653 345 653 274V107C653 36 642 32 583 26V0H797Z"></path></g><g transform="matrix(.013,0,0,-0.013,57.584,0)"><path d="M380 106C343 72 306 56 265 56C195 56 116 112 115 248C235 252 361 262 377 265C396 269 400 277 400 297C400 374 333 449 250 449H249C198 449 144 421 103 376S37 269 37 201C37 88 109 -12 232 -12C263 -12 332 6 395 84L380 106ZM225 412C281 412 315 364 314 312C314 297 308 292 290 292C232 290 176 289 120 289C135 370 180 412 225 412Z"></path></g><g transform="matrix(.013,0,0,-0.013,63.109,0)"><path d="M298 36L289 62C276 55 253 45 228 45C202 45 169 60 169 141V397H276C289 405 292 426 282 437H169V574L155 576L90 509V437H45L17 408L21 397H90V107C90 28 125 -12 188 -12C198 -12 213 -8 230 1L298 36Z"></path></g><g transform="matrix(.013,0,0,-0.013,66.996,0)"><path d="M380 106C343 72 306 56 265 56C195 56 116 112 115 248C235 252 361 262 377 265C396 269 400 277 400 297C400 374 333 449 250 449H249C198 449 144 421 103 376S37 269 37 201C37 88 109 -12 232 -12C263 -12 332 6 395 84L380 106ZM225 412C281 412 315 364 314 312C314 297 308 292 290 292C232 290 176 289 120 289C135 370 180 412 225 412Z"></path></g><g transform="matrix(.013,0,0,-0.013,72.521,0)"><path d="M181 342V451C133 431 89 419 40 411V388C98 381 102 377 102 311V104C102 38 95 32 33 26V0H263V26C186 32 181 38 181 104V287C203 343 235 372 261 372C277 372 289 366 304 352C310 346 318 345 330 350C349 359 362 379 362 399C362 422 338 449 304 449C256 449 213 393 183 342H181Z"></path></g><g transform="matrix(.013,0,0,-0.013,77.266,0)"><path d="M319 325C317 349 306 409 297 431C277 440 250 449 209 449C117 449 57 389 57 319C57 243 122 209 182 182C232 159 261 135 261 91C261 48 227 21 190 21C130 21 85 79 68 145L41 140C41 104 51 36 58 22C75 7 121 -12 172 -12C252 -12 337 35 337 126C337 195 286 231 210 262C166 281 126 304 126 348C126 388 152 417 191 417C240 417 274 378 294 318L319 325Z"></path></g><g transform="matrix(.013,0,0,-0.013,85.887,0)"><path d="M535 323V373H52V323H535ZM535 138V188H52V138H535Z"></path></g><g transform="matrix(.013,0,0,-0.013,97.15,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,103.39,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,106.354,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,112.594,0)"><path d="M95 130C70 130 46 113 46 88C46 72 54 64 59 64C93 55 121 33 121 -3C121 -41 93 -68 44 -88L55 -117C117 -98 186 -56 186 22C186 91 131 130 95 130Z"></path></g><g transform="matrix(.013,0,0,-0.013,117.737,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,123.977,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,126.941,0)"><path d="M412 140C382 77 369 73 315 73H129L270 222C362 320 402 379 402 466C402 571 322 635 234 635C177 635 130 609 99 576L42 495L64 475C90 514 133 568 201 568C274 568 318 519 318 435C318 349 255 267 193 193C144 135 87 78 32 23V0H405C417 45 427 89 440 131L412 140Z"></path></g><g transform="matrix(.013,0,0,-0.013,133.181,0)"><path d="M95 130C70 130 46 113 46 88C46 72 54 64 59 64C93 55 121 33 121 -3C121 -41 93 -68 44 -88L55 -117C117 -98 186 -56 186 22C186 91 131 130 95 130Z"></path></g><g transform="matrix(.013,0,0,-0.013,138.324,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,144.564,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,147.528,0)"><path d="M285 378C315 398 338 416 353 432C373 451 384 474 384 503C384 579 325 635 236 635H235C182 635 136 610 108 579L65 516L85 496C110 533 150 575 205 575C258 575 300 543 300 481C300 407 232 369 141 339L147 310C163 315 188 321 211 321C268 321 338 284 338 192C338 94 288 40 217 40C160 40 119 68 93 91C85 98 77 97 69 91C60 84 47 71 46 58C44 46 48 35 62 22C75 10 116 -12 162 -12C234 -12 424 62 424 224C424 297 373 359 285 376V378Z"></path></g><g transform="matrix(.013,0,0,-0.013,153.769,0)"><path d="M95 130C70 130 46 113 46 88C46 72 54 64 59 64C93 55 121 33 121 -3C121 -41 93 -68 44 -88L55 -117C117 -98 186 -56 186 22C186 91 131 130 95 130Z"></path></g><g transform="matrix(.013,0,0,-0.013,158.912,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,165.152,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,168.116,0)"><path d="M456 178V225H360V632H320C217 496 115 347 20 206V178H280V106C280 40 276 34 189 27V0H445V27C364 34 360 39 360 106V178H456ZM280 225H82C149 335 214 431 278 520H280V225Z"></path></g><g transform="matrix(.013,0,0,-0.013,174.356,0)"><path d="M95 130C70 130 46 113 46 88C46 72 54 64 59 64C93 55 121 33 121 -3C121 -41 93 -68 44 -88L55 -117C117 -98 186 -56 186 22C186 91 131 130 95 130Z"></path></g><g transform="matrix(.013,0,0,-0.013,179.499,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,185.739,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,188.703,0)"><path d="M153 550H386L412 615L406 623H120L82 318C104 327 142 338 184 338C294 338 347 275 347 187C347 112 305 39 221 39C160 39 119 71 97 89C88 97 80 96 71 90C59 80 50 67 49 57C48 45 52 36 66 23C80 9 123 -12 169 -12C221 -11 288 15 342 59C403 109 431 165 431 225C431 308 366 395 238 395C212 395 165 379 127 364L153 550Z"></path></g><g transform="matrix(.013,0,0,-0.013,194.943,0)"><path d="M95 130C70 130 46 113 46 88C46 72 54 64 59 64C93 55 121 33 121 -3C121 -41 93 -68 44 -88L55 -117C117 -98 186 -56 186 22C186 91 131 130 95 130Z"></path></g><g transform="matrix(.013,0,0,-0.013,200.086,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,206.326,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,209.29,0)"><path d="M137 343C167 482 260 545 321 574C357 591 397 603 429 609L423 641C382 634 335 622 295 608C189 570 37 457 37 238C37 84 125 -12 242 -12C362 -12 447 89 447 209C447 311 374 393 267 393C247 393 226 386 204 376L137 343ZM227 337C318 337 361 256 361 173C361 105 336 22 258 22C176 22 126 120 126 240C126 266 127 291 132 310C155 323 189 337 227 337Z"></path></g><g transform="matrix(.013,0,0,-0.013,215.531,0)"><path d="M95 130C70 130 46 113 46 88C46 72 54 64 59 64C93 55 121 33 121 -3C121 -41 93 -68 44 -88L55 -117C117 -98 186 -56 186 22C186 91 131 130 95 130Z"></path></g><g transform="matrix(.013,0,0,-0.013,220.674,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,226.914,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,229.878,0)"><path d="M447 623H65C61 580 56 530 47 475H76C100 541 106 550 172 550H388C308 376 196 170 91 -1L98 -12L172 -2C268 204 360 408 455 611L447 623Z"></path></g><g transform="matrix(.013,0,0,-0.013,236.12,0)"><path d="M95 130C70 130 46 113 46 88C46 72 54 64 59 64C93 55 121 33 121 -3C121 -41 93 -68 44 -88L55 -117C117 -98 186 -56 186 22C186 91 131 130 95 130Z"></path></g><g transform="matrix(.013,0,0,-0.013,241.263,0)"><path d="M433 39L423 65C413 59 399 54 387 54C370 54 352 69 352 114V299C352 352 342 392 307 422C285 440 255 449 225 449C168 437 102 399 75 379C56 365 44 353 44 339C44 315 69 296 87 296C101 296 111 303 116 319C124 349 133 371 145 385C156 397 171 404 190 404C241 404 275 364 275 291V274C253 256 180 229 120 209C65 190 39 159 39 110C39 47 88 -12 159 -12C189 -12 237 25 277 52C282 35 288 21 301 8C312 -3 333 -12 348 -12L433 39ZM275 84C256 65 221 48 195 48C164 48 124 73 124 124C124 161 146 180 185 198C206 208 254 229 275 240V84Z"></path></g><g transform="matrix(.013,0,0,-0.013,246.853,0)"><path d="M524 0V26C466 32 460 36 460 104V297C460 393 411 449 331 449C302 449 276 437 248 419C223 402 201 387 181 372V451C137 432 90 420 42 411V388C96 378 102 374 102 310V104C102 38 97 33 29 26V0H246V26C187 32 181 36 181 104V339C211 365 250 390 290 390C357 390 381 345 381 276V109C381 40 374 32 315 26V0H524Z"></path></g><g transform="matrix(.013,0,0,-0.013,253.911,0)"><path d="M517 51L485 54C448 58 441 63 441 115V712C404 700 337 684 285 678V653C357 648 362 645 362 580V437C339 446 309 449 295 449C159 449 38 340 38 201C38 61 143 -12 223 -12C234 -12 261 -6 301 17L362 53V-12C420 9 495 22 517 26V51ZM362 85C338 67 301 51 266 51C201 51 128 109 128 228C128 373 212 411 259 411C296 411 338 395 362 360V85Z"></path></g><g transform="matrix(.013,0,0,-0.013,263.025,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,269.265,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,272.229,0)"><path d="M249 635C141 635 70 555 70 471C70 401 114 353 179 316C143 294 106 267 90 252C68 231 45 202 45 157C45 50 130 -12 237 -12C322 -12 435 52 435 169C435 256 372 304 303 343C349 374 375 398 383 407C401 429 411 458 411 487C411 569 344 635 249 635ZM238 603C285 603 337 567 337 482C337 422 310 385 276 358C205 393 145 426 145 500C145 552 179 603 238 603ZM248 20C183 20 125 70 125 163C125 218 158 268 206 300C284 261 355 217 355 143C355 66 308 20 248 20Z"></path></g></svg>.</span></div>

Wireless Communications and Mobile Computing

fig6

Figure 6

Figure 6: A Smart Cache Content Update Policy Based on Deep Reinforcement Learning 