Adaptive Optimization of Traffic Signal Timing via Deep Reinforcement Learning

<table class="table-group" id="tab1"><tr><td><table class="table"><tr><td class="thead-hr" colspan="3"><hr/></td></tr><tr class="thead"><td class="align_left">Traffic flow direction</td><td class="align_center">One-hot coding</td><td class="align_center">Action code</td></tr><tr><td class="thead-hr" colspan="3"><hr/></td></tr><tr><td class="align_left">NS-straight</td><td class="align_center"><span style="width: 56.9734ptpx;"><svg height="12.5794pt" id="M18" style="vertical-align:-3.29107pt" version="1.1" viewbox="-0.0498162 -9.28833 56.9734 12.5794" width="56.9734pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M290 -163V-135C183 -126 181 -122 181 -44V583C181 662 184 666 290 675V703H120V-163H290Z"></path></g><g transform="matrix(.013,0,0,-0.013,4.485,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,10.725,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,19.144,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,24.287,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,29.43,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,34.607,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,40.847,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,47.087,0)"><path d="M226 -163V703H56V676C162 667 165 662 165 584V-43C165 -122 162 -126 56 -136V-163H226Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,51.572,3.132)"><path d="M504 89L488 119C453 85 428 69 416 69C408 69 405 75 412 102L462 304C492 438 460 451 436 451S388 441 356 423C311 397 229 336 170 256H168L189 340C208 418 200 451 177 451C144 451 82 413 24 352L40 322C65 345 98 369 108 369C114 369 119 363 112 335L28 -3L36 -12C54 -3 83 4 112 10C125 73 138 122 153 174C205 258 328 382 376 382C395 382 399 369 384 305L330 93C312 25 323 -12 351 -12C378 -12 439 21 504 89Z"></path></g></svg></span></td><td class="align_center"><span style="width: 11.0615ptpx;"><svg height="9.25202pt" id="M19" style="vertical-align:-3.29111pt" version="1.1" viewbox="-0.0498162 -5.96091 11.0615 9.25202" width="11.0615pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M483 97L471 123C436 91 401 65 392 65C388 65 384 74 390 106C414 239 444 378 457 429L455 433C444 433 429 436 416 439C392 444 368 448 344 448C281 448 204 415 152 376C71 315 23 205 23 103C23 21 57 -12 85 -12C114 -12 149 6 185 34C231 70 285 119 329 183H331L309 81C292 0 308 -12 326 -12C350 -12 421 24 483 97ZM374 387C370 363 356 291 345 261C315 193 181 50 139 50C124 50 110 71 110 118C110 224 153 331 218 379C238 394 271 402 301 402C329 402 359 394 374 387Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,5.993,3.132)"><path d="M245 635C92 635 37 457 37 312C37 149 91 -12 244 -12C395 -12 449 166 449 312C449 469 395 635 245 635ZM243 598C332 598 358 454 358 312C358 173 334 26 245 26C158 26 128 174 128 313S152 598 243 598Z"></path></g></svg></span></td></tr><tr><td class="align_left">NE-left; SW-left</td><td class="align_center"><span style="width: 56.9734ptpx;"><svg height="12.5794pt" id="M20" style="vertical-align:-3.29107pt" version="1.1" viewbox="-0.0498162 -9.28833 56.9734 12.5794" width="56.9734pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M290 -163V-135C183 -126 181 -122 181 -44V583C181 662 184 666 290 675V703H120V-163H290Z"></path></g><g transform="matrix(.013,0,0,-0.013,4.485,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,10.725,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,19.144,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,24.287,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,29.43,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,34.607,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,40.847,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,47.087,0)"><path d="M226 -163V703H56V676C162 667 165 662 165 584V-43C165 -122 162 -126 56 -136V-163H226Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,51.572,3.132)"><path d="M504 89L488 119C453 85 428 69 416 69C408 69 405 75 412 102L462 304C492 438 460 451 436 451S388 441 356 423C311 397 229 336 170 256H168L189 340C208 418 200 451 177 451C144 451 82 413 24 352L40 322C65 345 98 369 108 369C114 369 119 363 112 335L28 -3L36 -12C54 -3 83 4 112 10C125 73 138 122 153 174C205 258 328 382 376 382C395 382 399 369 384 305L330 93C312 25 323 -12 351 -12C378 -12 439 21 504 89Z"></path></g></svg></span></td><td class="align_center"><span style="width: 11.0615ptpx;"><svg height="9.14241pt" id="M21" style="vertical-align:-3.1815pt" version="1.1" viewbox="-0.0498162 -5.96091 11.0615 9.14241" width="11.0615pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M483 97L471 123C436 91 401 65 392 65C388 65 384 74 390 106C414 239 444 378 457 429L455 433C444 433 429 436 416 439C392 444 368 448 344 448C281 448 204 415 152 376C71 315 23 205 23 103C23 21 57 -12 85 -12C114 -12 149 6 185 34C231 70 285 119 329 183H331L309 81C292 0 308 -12 326 -12C350 -12 421 24 483 97ZM374 387C370 363 356 291 345 261C315 193 181 50 139 50C124 50 110 71 110 118C110 224 153 331 218 379C238 394 271 402 301 402C329 402 359 394 374 387Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,5.993,3.132)"><path d="M389 0V32C297 38 291 46 291 118V635C234 613 175 595 109 583V556L161 554C203 552 207 547 207 497V118C207 46 201 38 110 32V0H389Z"></path></g></svg></span></td></tr><tr><td class="align_left">…</td><td class="align_center">…</td><td class="align_center">…</td></tr><tr><td class="align_left">EW-straight</td><td class="align_center"><span style="width: 56.9734ptpx;"><svg height="12.5794pt" id="M22" style="vertical-align:-3.29107pt" version="1.1" viewbox="-0.0498162 -9.28833 56.9734 12.5794" width="56.9734pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M290 -163V-135C183 -126 181 -122 181 -44V583C181 662 184 666 290 675V703H120V-163H290Z"></path></g><g transform="matrix(.013,0,0,-0.013,4.485,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,10.725,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,19.144,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,24.287,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,29.43,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,34.607,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,40.847,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,47.087,0)"><path d="M226 -163V703H56V676C162 667 165 662 165 584V-43C165 -122 162 -126 56 -136V-163H226Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,51.572,3.132)"><path d="M504 89L488 119C453 85 428 69 416 69C408 69 405 75 412 102L462 304C492 438 460 451 436 451S388 441 356 423C311 397 229 336 170 256H168L189 340C208 418 200 451 177 451C144 451 82 413 24 352L40 322C65 345 98 369 108 369C114 369 119 363 112 335L28 -3L36 -12C54 -3 83 4 112 10C125 73 138 122 153 174C205 258 328 382 376 382C395 382 399 369 384 305L330 93C312 25 323 -12 351 -12C378 -12 439 21 504 89Z"></path></g></svg></span></td><td class="align_center"><span style="width: 21.3191ptpx;"><svg height="9.36162pt" id="M23" style="vertical-align:-3.40071pt" version="1.1" viewbox="-0.0498162 -5.96091 21.3191 9.36162" width="21.3191pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M483 97L471 123C436 91 401 65 392 65C388 65 384 74 390 106C414 239 444 378 457 429L455 433C444 433 429 436 416 439C392 444 368 448 344 448C281 448 204 415 152 376C71 315 23 205 23 103C23 21 57 -12 85 -12C114 -12 149 6 185 34C231 70 285 119 329 183H331L309 81C292 0 308 -12 326 -12C350 -12 421 24 483 97ZM374 387C370 363 356 291 345 261C315 193 181 50 139 50C124 50 110 71 110 118C110 224 153 331 218 379C238 394 271 402 301 402C329 402 359 394 374 387Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,5.993,3.132)"><path d="M504 89L488 119C453 85 428 69 416 69C408 69 405 75 412 102L462 304C492 438 460 451 436 451S388 441 356 423C311 397 229 336 170 256H168L189 340C208 418 200 451 177 451C144 451 82 413 24 352L40 322C65 345 98 369 108 369C114 369 119 363 112 335L28 -3L36 -12C54 -3 83 4 112 10C125 73 138 122 153 174C205 258 328 382 376 382C395 382 399 369 384 305L330 93C312 25 323 -12 351 -12C378 -12 439 21 504 89Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,10.652,3.132)"><path d="M556 236V289H56V236H556Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,16.212,3.132)"><path d="M389 0V32C297 38 291 46 291 118V635C234 613 175 595 109 583V556L161 554C203 552 207 547 207 497V118C207 46 201 38 110 32V0H389Z"></path></g></svg></span></td></tr><tr><td class="align_left">ES-left; WN-left</td><td class="align_center"><span style="width: 56.9734ptpx;"><svg height="12.5794pt" id="M24" style="vertical-align:-3.29107pt" version="1.1" viewbox="-0.0498162 -9.28833 56.9734 12.5794" width="56.9734pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M290 -163V-135C183 -126 181 -122 181 -44V583C181 662 184 666 290 675V703H120V-163H290Z"></path></g><g transform="matrix(.013,0,0,-0.013,4.485,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,10.725,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,19.144,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,24.287,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,29.43,0)"><path d="M113 -12C146 -12 170 11 170 46C170 78 146 103 114 103S58 78 58 46C58 11 82 -12 113 -12Z"></path></g><g transform="matrix(.013,0,0,-0.013,34.607,0)"><path d="M241 635C89 635 35 457 35 312C35 153 89 -12 240 -12C390 -12 443 166 443 312C443 466 390 635 241 635ZM238 602C329 602 354 454 354 312C354 172 330 22 240 22C152 22 124 173 124 313S148 602 238 602Z"></path></g><g transform="matrix(.013,0,0,-0.013,40.847,0)"><path d="M384 0V27C293 34 287 42 287 114V635C232 613 172 594 109 583V559L157 557C201 555 205 550 205 499V114C205 42 199 34 109 27V0H384Z"></path></g><g transform="matrix(.013,0,0,-0.013,47.087,0)"><path d="M226 -163V703H56V676C162 667 165 662 165 584V-43C165 -122 162 -126 56 -136V-163H226Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,51.572,3.132)"><path d="M504 89L488 119C453 85 428 69 416 69C408 69 405 75 412 102L462 304C492 438 460 451 436 451S388 441 356 423C311 397 229 336 170 256H168L189 340C208 418 200 451 177 451C144 451 82 413 24 352L40 322C65 345 98 369 108 369C114 369 119 363 112 335L28 -3L36 -12C54 -3 83 4 112 10C125 73 138 122 153 174C205 258 328 382 376 382C395 382 399 369 384 305L330 93C312 25 323 -12 351 -12C378 -12 439 21 504 89Z"></path></g></svg></span></td><td class="align_center"><span style="width: 11.2899ptpx;"><svg height="9.25202pt" id="M25" style="vertical-align:-3.29111pt" version="1.1" viewbox="-0.0498162 -5.96091 11.2899 9.25202" width="11.2899pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g transform="matrix(.013,0,0,-0.013,0,0)"><path d="M483 97L471 123C436 91 401 65 392 65C388 65 384 74 390 106C414 239 444 378 457 429L455 433C444 433 429 436 416 439C392 444 368 448 344 448C281 448 204 415 152 376C71 315 23 205 23 103C23 21 57 -12 85 -12C114 -12 149 6 185 34C231 70 285 119 329 183H331L309 81C292 0 308 -12 326 -12C350 -12 421 24 483 97ZM374 387C370 363 356 291 345 261C315 193 181 50 139 50C124 50 110 71 110 118C110 224 153 331 218 379C238 394 271 402 301 402C329 402 359 394 374 387Z"></path></g><g transform="matrix(.0091,0,0,-0.0091,5.993,3.132)"><path d="M504 89L488 119C453 85 428 69 416 69C408 69 405 75 412 102L462 304C492 438 460 451 436 451S388 441 356 423C311 397 229 336 170 256H168L189 340C208 418 200 451 177 451C144 451 82 413 24 352L40 322C65 345 98 369 108 369C114 369 119 363 112 335L28 -3L36 -12C54 -3 83 4 112 10C125 73 138 122 153 174C205 258 328 382 376 382C395 382 399 369 384 305L330 93C312 25 323 -12 351 -12C378 -12 439 21 504 89Z"></path></g></svg></span></td></tr><tr class="table-tr"><td colspan="3"><hr class="tbody-hr"/></td></tr></table></td></tr></table>

<div>Green light phase coding table.</div>

Journal of Advanced Transportation

tab1

Table 1

Table 1: Adaptive Optimization of Traffic Signal Timing via Deep Reinforcement Learning 