Edge Caching for D2D Enabled Hierarchical Wireless Networks with Deep Reinforcement Learning

<table class="fixed-width table-group" id="tab1"><tr><td><table class="table"><colgroup><col style="width:3.78em"/><col style="width:3.11em"/><col style="width:4.93em"/><col style="width:4.90em"/><col style="width:3.25em"/><col style="width:2.75em"/><col style="width:2.49em"/><col style="width:2.34em"/><col style="width:2.95em"/><col style="width:2.71em"/></colgroup><tr><td class="thead-hr" colspan="10"><hr/></td></tr><tr class="thead"><td class="align_left"> </td><td class="align_center"><span style="width: 8.02022ptpx;"><svg height="8.68572pt" id="M241" style="vertical-align:-0.0498209pt" version="1.1" viewbox="-0.0498162 -8.6359 8.02022 8.68572" width="8.02022pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M584 650H137L131 622C214 614 217 612 200 521L125 127C109 41 101 35 23 28L17 0H288L294 28C201 35 193 42 209 128L242 309H348C440 309 442 300 443 226H471L510 422H482C452 354 449 348 357 348H251L295 575C302 609 304 615 338 615H426C502 615 517 604 526 581C534 560 536 524 537 492L565 494C574 554 583 631 584 650Z"></path></g></svg></span></td><td class="align_center"><span style="width: 47.6623ptpx;"><svg height="11.439pt" id="M242" style="vertical-align:-2.15067pt" version="1.1" viewbox="-0.0498162 -9.28833 47.6623 11.439" width="47.6623pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M689 332C689 394 670 448 646 448C620 448 597 421 597 396C597 386 600 381 608 372C619 359 620 334 620 315C620 150 538 45 454 45C414 45 386 67 386 122C386 138 388 158 394 180L457 426L452 432L377 416L315 156C302 100 259 45 216 45C176 45 148 67 148 122C148 133 152 158 156 180C162 212 173 259 194 332C201 357 206 384 206 405C206 430 198 448 174 448C125 448 66 406 23 342L43 319C84 368 110 383 121 383C126 383 128 382 128 377C128 370 127 359 122 343C99 268 84 204 77 156C74 137 70 111 70 104C70 25 125 -12 180 -12C228 -12 276 12 319 50C338 8 378 -12 418 -12C549 -12 689 166 689 332Z"></path></g><g transform="matrix(.013,0,0,-0.013,9.257,0)"><path d="M290 -163V-135C183 -126 181 -122 181 -44V583C181 662 184 666 290 675V703H120V-163H290Z"></path></g><g transform="matrix(.013,0,0,-0.013,13.742,0)"><path d="M962 650H795L470 145L347 650H176L170 622C268 613 275 606 244 503L190 322C150 188 132 126 118 91C102 50 80 33 18 28L12 0H245L251 28C175 35 170 48 174 93C177 128 191 180 220 284L292 542H294C331 392 383 150 409 4H432L774 555H776L714 137C700 40 694 34 612 28L606 0H868L874 28C793 34 784 37 797 137L849 533C859 612 863 616 956 622L962 650Z"></path></g><g transform="matrix(.013,0,0,-0.013,25.897,0)"><path d="M828 650H570L564 622C646 614 650 609 634 524L604 366H253L281 524C296 608 308 615 382 622L388 650H132L126 622C211 615 214 609 198 524L121 122C106 42 102 35 23 28L17 0H276L282 28C196 35 191 40 206 122L243 324H597L559 123C544 42 537 35 452 28L446 0H710L716 28C632 35 628 43 643 122L719 524C735 610 741 615 822 622L828 650Z"></path></g><g transform="matrix(.013,0,0,-0.013,36.429,0)"><path d="M475 445L452 468C423 436 401 426 385 426S352 427 305 436C268 443 216 448 185 448C156 448 128 425 102 382C89 360 77 338 64 304L89 291C117 340 143 380 190 380C214 380 268 376 295 373C320 370 344 371 362 374L70 101C32 65 23 36 23 18C23 5 27 -3 31 -8C36 -8 39 -6 42 -1C52 15 67 31 80 40S110 49 143 33C195 8 257 -20 290 -20C339 -20 382 11 439 129L416 148C366 74 334 55 305 55C278 55 236 68 203 81C169 94 141 100 122 96C205 170 348 308 411 376L475 445Z"></path></g><g transform="matrix(.013,0,0,-0.013,42.904,0)"><path d="M226 -163V703H56V676C162 667 165 662 165 584V-43C165 -122 162 -126 56 -136V-163H226Z"></path></g></svg></span></td><td class="align_center"><span style="width: 47.1935ptpx;"><svg height="13.7421pt" id="M243" style="vertical-align:-2.1507pt" version="1.1" viewbox="-0.0498162 -11.5914 47.1935 13.7421" width="47.1935pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M548 455L523 469C505 448 490 439 461 439C416 439 374 447 316 447C144 447 23 310 23 161C23 47 89 -12 179 -12C312 -12 429 106 429 244C429 303 419 344 368 386L371 388C404 383 440 378 474 378C507 378 530 411 548 455ZM350 274C350 191 308 25 198 25C144 25 108 75 108 157C108 264 170 395 276 395C296 395 320 384 333 360C347 334 350 316 350 274Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,7.347,-5.741)"><path d="M414 144C384 79 371 75 317 75H135L276 221C367 316 408 376 408 465C408 570 327 635 237 635C179 635 131 609 100 575L42 494L67 471C94 510 138 565 205 565C277 565 321 517 321 435C321 348 258 270 195 195C146 137 88 81 33 26V0H411C423 44 433 88 446 135L414 144Z"></path></g><g transform="matrix(.013,0,0,-0.013,12.294,0)"><path d="M290 -163V-135C183 -126 181 -122 181 -44V583C181 662 184 666 290 675V703H120V-163H290Z"></path></g><g transform="matrix(.013,0,0,-0.013,16.779,0)"><path d="M530 686C535 705 530 712 521 712C504 712 448 684 359 674L358 648H393C437 648 439 646 429 593L400 435C372 447 345 448 332 448C286 448 194 414 144 373C68 311 23 203 23 111C23 26 57 -12 91 -12C120 -12 147 3 188 29C227 54 290 102 341 170H343L322 71C308 6 320 -12 341 -12C373 -12 442 27 501 96L485 120C455 91 422 67 408 67C401 67 401 76 404 91C440 294 479 473 530 686ZM387 375L355 241C326 187 200 53 142 53C126 53 109 73 109 130C109 217 154 337 218 381C240 396 265 404 297 404S372 390 387 375Z"></path></g><g transform="matrix(.013,0,0,-0.013,23.994,0)"><path d="M578 512C578 619 494 650 387 650H140L134 622C219 615 223 609 210 542L127 119C112 40 104 34 23 28L17 0H235C317 0 387 7 444 33C515 65 564 122 564 195C564 289 495 335 412 352V354C505 373 578 422 578 512ZM486 510C486 422 423 367 314 367H257L294 565C299 591 305 602 315 608C324 614 339 617 362 617C421 617 486 595 486 510ZM466 200C466 100 393 35 296 35C222 35 198 51 212 127L250 333H303C388 333 466 303 466 200Z"></path></g><g transform="matrix(.013,0,0,-0.013,32.199,0)"><path d="M766 88L752 113C719 83 690 64 681 64C674 64 672 74 679 103L724 292C758 436 724 448 701 448C680 448 666 442 639 429C594 407 514 350 441 252H439L447 289C476 423 450 448 419 448C398 448 379 441 355 427C307 400 234 344 162 249H160L180 324C203 409 197 448 170 448C144 448 82 413 23 349L35 321C57 343 96 374 108 374C115 374 117 371 111 341C87 227 57 112 24 -6L32 -12C53 -4 81 4 108 6C119 68 134 128 149 171C177 229 309 383 364 383C387 383 388 355 373 282C354 190 330 92 303 -6L309 -12C332 -4 356 3 386 6C396 63 411 122 424 171C458 236 590 383 642 383C658 383 664 369 652 315L603 91C587 20 593 -12 619 -12C642 -12 708 23 766 88Z"></path></g><g transform="matrix(.013,0,0,-0.013,42.456,0)"><path d="M226 -163V703H56V676C162 667 165 662 165 584V-43C165 -122 162 -126 56 -136V-163H226Z"></path></g></svg></span></td><td class="align_center"><span style="width: 14.0879ptpx;"><svg height="9.09021pt" id="M244" style="vertical-align:-0.2455397pt" version="1.1" viewbox="-0.0498162 -8.84467 14.0879 9.09021" width="14.0879pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M1048 672C1036 675 1028 674 1020 674C822 646 666 318 554 134C521 81 495 39 468 39C457 39 455 46 455 57C455 79 469 109 472 118C520 227 653 461 794 660L779 672C768 675 766 674 762 674C654 674 544 586 344 250C241 78 187 15 119 15C104 15 91 21 88 36H91C96 33 101 32 106 32C133 32 149 53 149 76C149 103 128 128 97 128C65 128 38 102 38 63C38 20 71 -15 118 -15C235 -15 307 132 473 397C542 508 638 615 712 644L713 643C646 565 574 455 521 373C460 281 378 144 378 66C378 23 399 -4 434 -4C518 -4 559 76 647 231C792 488 891 603 977 635L978 633C779 366 670 179 670 70C670 18 708 -12 756 -12C819 -12 883 42 959 134L940 150C868 66 810 18 770 18C751 18 737 28 737 53C737 111 865 359 1056 659L1048 672Z"></path></g></svg></span></td><td class="align_center"><span style="width: 17.8834ptpx;"><svg height="11.6817pt" id="M245" style="vertical-align:-0.2455997pt" version="1.1" viewbox="-0.0498162 -11.4361 17.8834 11.6817" width="17.8834pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M1048 672C1036 675 1028 674 1020 674C822 646 666 318 554 134C521 81 495 39 468 39C457 39 455 46 455 57C455 79 469 109 472 118C520 227 653 461 794 660L779 672C768 675 766 674 762 674C654 674 544 586 344 250C241 78 187 15 119 15C104 15 91 21 88 36H91C96 33 101 32 106 32C133 32 149 53 149 76C149 103 128 128 97 128C65 128 38 102 38 63C38 20 71 -15 118 -15C235 -15 307 132 473 397C542 508 638 615 712 644L713 643C646 565 574 455 521 373C460 281 378 144 378 66C378 23 399 -4 434 -4C518 -4 559 76 647 231C792 488 891 603 977 635L978 633C779 366 670 179 670 70C670 18 708 -12 756 -12C819 -12 883 42 959 134L940 150C868 66 810 18 770 18C751 18 737 28 737 53C737 111 865 359 1056 659L1048 672Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,13.936,-5.741)"><path d="M310 541L304 571C290 586 211 619 185 610L80 76L131 52L310 541Z"></path></g></svg></span></td><td class="align_center"><span style="width: 6.63704ptpx;"><svg height="9.39034pt" id="M246" style="vertical-align:-3.42943pt" version="1.1" viewbox="-0.0498162 -5.96091 6.63704 9.39034" width="6.63704pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M478 372C478 418 458 448 431 448C409 448 389 431 389 410C389 404 391 400 394 395C398 388 406 371 406 348C406 253 308 122 251 51H249C254 122 249 257 231 336C212 421 189 448 159 448C126 448 75 412 23 327L48 306C83 354 103 371 115 371C125 371 134 360 144 334C185 224 192 64 183 -19C146 -100 116 -202 110 -244L125 -261C154 -259 208 -234 222 -220C222 -194 225 -84 235 -23C247 -3 273 36 308 79C379 165 478 288 478 372Z"></path></g></svg></span></td><td class="align_center"><span style="width: 5.63229ptpx;"><svg height="6.1673pt" id="M247" style="vertical-align:-0.2063904pt" version="1.1" viewbox="-0.0498162 -5.96091 5.63229 6.1673" width="5.63229pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M401 397C401 420 368 448 302 448C245 448 169 416 122 377C62 327 23 254 23 169C23 45 83 -12 181 -12C252 -12 323 29 374 85L358 107C305 62 257 43 210 43C147 43 110 98 110 189V214L313 208L321 256L115 250C132 342 190 405 253 405C291 405 323 389 346 360C356 348 364 348 377 357C392 367 401 384 401 397Z"></path></g></svg></span></td><td class="align_center"><span style="width: 7.51131ptpx;"><svg height="6.1673pt" id="M248" style="vertical-align:-0.2063904pt" version="1.1" viewbox="-0.0498162 -5.96091 7.51131 6.1673" width="7.51131pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M545 106L524 126C493 85 467 65 455 65C438 65 427 113 405 238C448 295 498 362 543 439L533 448L478 435C453 386 423 331 398 295H395C370 404 347 448 282 448C169 448 23 309 23 153C23 54 65 -12 128 -12C203 -12 283 70 339 155H341C360 29 380 -12 411 -12C444 -12 491 11 545 106ZM333 204C265 95 210 54 169 54C137 54 113 96 113 171C113 302 191 405 252 405C301 405 318 306 333 204Z"></path></g></svg></span></td><td class="align_center"><span style="width: 7.47216ptpx;"><svg height="12.3916pt" id="M249" style="vertical-align:-3.42948pt" version="1.1" viewbox="-0.0498162 -8.96212 7.47216 12.3916" width="7.47216pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M542 242C542 369 469 429 379 468C370 472 372 496 376 513L416 681L400 691L344 648L219 51C167 69 113 128 113 217C113 324 166 387 266 428L252 459C124 418 23 336 23 202C23 95 90 13 205 -8L182 -132C165 -222 191 -254 209 -261L215 -258L270 -7C415 -4 542 98 542 242ZM469 232C469 126 403 33 279 39L355 409C399 394 469 331 469 232Z"></path></g></svg></span></td></tr><tr><td class="thead-hr" colspan="10"><hr/></td></tr><tr><td class="align_left">Range</td><td class="align_center">1000</td><td class="align_center">200</td><td class="align_center">-95</td><td class="align_center">5000</td><td class="align_center">200</td><td class="align_center">0.9</td><td class="align_center">0.1</td><td class="align_center">0.05</td><td class="align_center">250</td></tr><tr class="table-tr"><td colspan="10"><hr class="tbody-hr"/></td></tr></table></td></tr></table>

Wireless Communications and Mobile Computing

tab1

Table 1

Table 1: Edge Caching for D2D Enabled Hierarchical Wireless Networks with Deep Reinforcement Learning 